wordlist 0.1.0 → 0.1.1

data/lib/wordlist/parsers.rb

@@ -1,34 +1,39 @@
 module Wordlist
   module Parsers
-    def self.included(base)
-      base.module_eval do
-        # Ignore case of parsed text
-        attr_accessor :ignore_case
+    # Ignore case of parsed text
+    attr_accessor :ignore_case
 
-        # Ignore the punctuation of parsed text
-        attr_accessor :ignore_punctuation
+    # Ignore the punctuation of parsed text
+    attr_accessor :ignore_punctuation
 
-        # Ignore URLs
-        attr_accessor :ignore_urls
+    # Ignore URLs
+    attr_accessor :ignore_urls
 
-        # Ignore Phone numbers
-        attr_accessor :ignore_phone_numbers
+    # Ignore Phone numbers
+    attr_accessor :ignore_phone_numbers
 
-        # Ignore References
-        attr_accessor :ignore_references
-      end
-    end
+    # Ignore References
+    attr_accessor :ignore_references
 
+    #
+    # Initializes the parsers settings.
+    #
     def initialize
-      @ignore_case = false
-      @ignore_punctuation = true
-      @ignore_urls = true
+      @ignore_case          = false
+      @ignore_punctuation   = true
+      @ignore_urls          = true
       @ignore_phone_numbers = false
-      @ignore_references = false
+      @ignore_references    = false
     end
 
     #
-    # Parses the specified _text_ and returns an Array of tokens.
+    # Parses the given text.
+    #
+    # @param [String] text
+    #   The text to parse.
+    #
+    # @return [Array<String>]
+    #   The Array of parsed tokens.
     #
     def parse(text)
       text = text.to_s

data/lib/wordlist/runners.rb

@@ -0,0 +1,2 @@
+require 'wordlist/runners/runner'
+require 'wordlist/runners/list'

data/lib/wordlist/runners/list.rb

@@ -0,0 +1,116 @@
+require 'wordlist/runners/runner'
+require 'wordlist/flat_file'
+
+module Wordlist
+  module Runners
+    class List < Runner
+
+      #
+      # Creates a new List Runner.
+      #
+      def initialize
+        @file = nil
+        @min_length = nil
+        @max_length = nil
+        @mutations = []
+
+        @words = false
+        @unique_words = false
+
+        @output = nil
+      end
+
+      #
+      # Runs the list runner.
+      #
+      # @param [Array<String>] args
+      #   Arguments to run the runner with.
+      #
+      def run(*args)
+        super(*args)
+
+        list = if @file
+                 FlatFile.new(
+                   @file,
+                   :min_length => @min_length,
+                   :max_length => @max_length
+                 )
+               else
+                 print_error('the --file option must be specified')
+                 exit -1
+               end
+
+        @mutations.each do |pattern,substitute|
+          list.mutate(pattern,substitute)
+        end
+
+        words = lambda { |output|
+          puts = output.method(:puts)
+
+          if @unique_words
+            list.each_unique(&puts)
+          elsif @words
+            list.each_word(&puts)
+          else
+            list.each(&puts)
+          end
+        }
+
+        if @output
+          File.open(@output,'w+',&words)
+        else
+          words.call(Kernel)
+        end
+      end
+
+      protected
+
+      #
+      # Parses the given arguments.
+      #
+      # @param [Array<String>] args
+      #   Arguments to parse.
+      #
+      def optparse(*args)
+        super(*args) do |opts|
+          opts.banner = 'usage: wordlist [options]'
+
+          opts.on('-f','--file FILE','The wordlist file to list') do |file|
+            @file = file
+          end
+
+          opts.on('--min-length NUM','Minimum length of words in characters') do |min|
+            @min_length = min
+          end
+
+          opts.on('--max-length NUM','Maximum length of words in characters') do |max|
+            @max_length = max
+          end
+
+          opts.on('-m','--mutate SUBSTRING::REPLACE','Adds a mutation rule') do |substring_and_replace|
+            @mutations << substring_and_replace.split('::',2)
+          end
+
+          opts.on('-M','--mutate-pattern PATTERN::REPLACE','Adds a mutation rule') do |pattern_and_replace|
+            pattern, replace = substring_and_replace.split('::',2)
+
+            @mutations << [Regexp.new(pattern), replace]
+          end
+
+          opts.on('-w','--words','Only print the words in the wordlist') do
+            @words = true
+          end
+
+          opts.on('-u','--unique','Only print the unique words in the wordlist') do
+            @unique_words = true
+          end
+
+          opts.on('-o','--output FILE','Optional file to output the wordlist to') do |file|
+            @output = File.expand_path(file)
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/wordlist/runners/runner.rb

@@ -0,0 +1,67 @@
+require 'optparse'
+
+module Wordlist
+  module Runners
+    class Runner
+      #
+      # Creates and runs the runner with the given arguments.
+      #
+      # @param [Array<String>] args
+      #   Arguments to parse.
+      #
+      def self.run(*args)
+        runner = new()
+        runner.run(*args)
+      end
+
+      #
+      # Runs the runner with the given arguments.
+      #
+      # @param [Array<String>] args
+      #   Arguments to run the runner with.
+      #
+      def run(*args)
+        optparse(*args)
+      end
+
+      protected
+
+      #
+      # Prints the given error message.
+      #
+      # @param [String] message
+      #   The error message to print.
+      #
+      def print_error(message)
+        $stderr.puts "#{$0}: #{message}"
+      end
+
+      #
+      # Parses the given arguments.
+      #
+      # @param [Array<String>] args
+      #   Arguments to parse.
+      #
+      # @yield [opts]
+      #   If a block is given, it will be passed the option parse to be
+      #   configured.
+      #
+      # @yieldparam [OptionParser] opts
+      #   The option parser to be configured.
+      #
+      def optparse(*args)
+        opts = OptionParser.new()
+
+        yield opts if block_given?
+
+        begin
+          opts.parse!(args)
+        rescue OptionParser::InvalidOption => e
+          $stderr.puts e.message
+          $stderr.puts opts
+          exit -1
+        end
+      end
+    end
+  end
+end

data/lib/wordlist/unique_filter.rb

@@ -14,8 +14,13 @@ module Wordlist
     end
 
     #
-    # Returns +true+ if the _word_ has been previously seen, returns
-    # +false+ otherwise.
+    # Determines if the given word has been previously seen.
+    #
+    # @param [String] word
+    #   The word to check for.
+    #
+    # @return [Boolean]
+    #   Specifies whether the word has been previously seen.
     #
     def seen?(word)
       length = word.length
@@ -24,12 +29,18 @@ module Wordlist
     end
 
     #
-    # Marks the specified _word_ as seen and returns +true+. If the _word_
-    # has been previously been seen, +false+ will be returned.
+    # Marks the given word as previously seen.
+    #
+    # @param [String] word
+    #   The word to mark as previously seen.
+    #
+    # @return [Boolean]
+    #   Specifies whether or not the word has not been previously seen
+    #   until now.
     #
     def saw!(word)
       length = word.length
-      crc = crc32(word)
+      crc    = crc32(word)
 
       if @seen.has_key?(length)
         return false if @seen[length].include?(crc)
@@ -42,8 +53,19 @@ module Wordlist
     end
 
     #
-    # Passes the specified _word_ through the unique filter, if the
-    # _word_ has not yet been seen, it will be passed to the given _block_.
+    # Passes the given word through the unique filter.
+    #
+    # @param [String] word
+    #   The word to pass through the unique filter.
+    #
+    # @yield [word]
+    #   The given block will be passed the word, if the word has not been
+    #   previously seen by the filter.
+    #
+    # @yieldparam [String] word
+    #   A unique word that has not been previously seen by the filter.
+    #
+    # @return [nil]
     #
     def pass(word)
       if saw!(word)
@@ -53,10 +75,27 @@ module Wordlist
       return nil
     end
 
+    #
+    # Clears the unique filter.
+    #
+    # @return [UniqueFilter]
+    #   The cleared filter.
+    #
+    def clear
+      @seen.clear
+      return self
+    end
+
     protected
 
     #
-    # Returns the CRC32 checksum of the specified _word_.
+    # Returns the CRC32 checksum of the given word.
+    #
+    # @param [String] word
+    #   The word to calculate a CRC32 checksum for.
+    #
+    # @return [Integer]
+    #   The CRC32 checksum for the given word.
     #
     def crc32(word)
       r = 0xffffffff

data/lib/wordlist/version.rb

@@ -1,4 +1,4 @@
 module Wordlist
   # Word version
-  VERSION = '0.1.0'
+  VERSION = '0.1.1'
 end

data/scripts/benchmark

@@ -1,7 +1,7 @@
 #!/usr/bin/env ruby
 $LOAD_PATH.unshift(File.expand_path(File.join(File.dirname(__FILE__),'..','lib')))
 
-require 'wordlist/builder'
+require 'wordlist'
 require 'benchmark'
 require 'fileutils'
 
@@ -10,9 +10,50 @@ path = File.expand_path(File.join(File.dirname(__FILE__),'shakespeare_wordlist.t
 FileUtils.rm_f(path)
 
 Benchmark.bm do |bm|
-  bm.report('build:') do
+  bm.report('build') do
     Wordlist::Builder.build(path) do |wordlist|
       wordlist.parse_file('/home/hal/shaks12.txt')
     end
   end
+
+  bm.report('each_unique') do
+    Wordlist::FlatFile.new(path) do |wordlist|
+      wordlist.each_unique { |word| word }
+    end
+  end
+
+  bm.report('each_mutation (1)') do
+    Wordlist::FlatFile.new(path) do |wordlist|
+      wordlist.mutate /o/i, '0'
+
+      wordlist.each_mutation { |word| word }
+    end
+  end
+
+  bm.report('each_mutation (2)') do
+    Wordlist::FlatFile.new(path) do |wordlist|
+      wordlist.mutate /o/i, '0'
+      wordlist.mutate /a/i, '@'
+
+      wordlist.each_mutation { |word| word }
+    end
+  end
+
+  bm.report('each_mutation (3)') do
+    Wordlist::FlatFile.new(path) do |wordlist|
+      wordlist.mutate /o/i, '0'
+      wordlist.mutate /a/i, '@'
+      wordlist.mutate /e/i, '3'
+
+      wordlist.each_mutation { |word| word }
+    end
+  end
+end
+
+Benchmark.bm do |bm|
+  mutator = Wordlist::Mutator.new(/o/i, '0')
+
+  bm.report('Mutator#each') do
+    mutator.each('lololololoLOLOLOLOLO') { |word| }
+  end
 end

data/spec/builder_examples.rb

@@ -0,0 +1,46 @@
+require 'spec_helper'
+require 'helpers/text'
+require 'helpers/wordlist'
+
+shared_examples_for "a wordlist Builder" do
+  include Helpers
+
+  before(:all) do
+    @words = ['dog', 'cat', 'catx', 'dat', 'dog', 'cat']
+    @sentence = 'dog cat catx, dog dat.'
+    @text = 'dog cat: catx. dog cat dat dog.'
+    @file = Helpers::SAMPLE_TEXT
+  end
+
+  it "should build a unique wordlist from words" do
+    Builder.build(@path) do |wordlist|
+      wordlist += @words
+    end
+
+    should_contain_words(@path,@expected)
+  end
+
+  it "should build a unique wordlist from a sentence" do
+    Builder.build(@path) do |wordlist|
+      wordlist.parse(@sentence)
+    end
+
+    should_contain_words(@path,@expected)
+  end
+
+  it "should build a unique wordlist from text" do
+    Builder.build(@path) do |wordlist|
+      wordlist.parse(@text)
+    end
+
+    should_contain_words(@path,@expected)
+  end
+
+  it "should build a unique wordlist from a file" do
+    Builder.build(@path) do |wordlist|
+      wordlist.parse_file(@file)
+    end
+
+    should_contain_words(@path,@expected)
+  end
+end