wordlist 0.1.1 → 1.0.0

data/lib/wordlist/list.rb

@@ -1,162 +0,0 @@
-require 'wordlist/unique_filter'
-require 'wordlist/mutator'
-
-module Wordlist
-  class List
-
-    include Enumerable
-
-    # Maximum length of words
-    attr_accessor :max_length
-
-    # Minimum length of words
-    attr_accessor :min_length
-
-    #
-    # Creates a new List object.
-    #
-    # @param [Hash] options
-    #   Additional options.
-    #
-    # @option options [Integer] :max_length
-    #   The maximum length of words produced by the list.
-    #
-    # @option options [Integer] :min_length
-    #   The minimum length of words produced by the list.
-    #
-    # @yield [list]
-    #   If a block is given, it will be passed the new list object.
-    #
-    # @yieldparam [List] list
-    #   The new list object.
-    #
-    def initialize(options={})
-      @mutators = []
-
-      @max_length = options[:max_length]
-      @min_length = options.fetch(:min_length,0)
-
-      yield self if block_given?
-    end
-
-    #
-    # Adds a mutation rule for the specified pattern, to be replaced
-    # using the specified substitute.
-    #
-    # @param [String, Regexp] pattern
-    #   The pattern to recognize text to mutate.
-    #
-    # @param [String, Integer, nil] substitute
-    #   The optional text to replace recognized text.
-    #
-    # @yield [match]
-    #   If a block is given, it will be passed the recognized text to be
-    #   mutated. The return value of the block will be used to replace
-    #   the recognized text.
-    #
-    # @yieldparam [String] match
-    #   The recognized text to be mutated.
-    #
-    # @example
-    #   list.mutate 'o', '0'
-    #   
-    #   list.mutate '0', 0x41
-    #   
-    #   list.mutate(/[oO]/) do |match|
-    #     match.swapcase
-    #   end
-    #
-    def mutate(pattern,substitute=nil,&block)
-      @mutators << Mutator.new(pattern,substitute,&block)
-    end
-
-    #
-    # Enumerate through every word in the list.
-    #
-    # @yield [word]
-    #   The given block will be passed each word in the list.
-    #
-    # @yieldparam [String] word
-    #   A word from the list.
-    #
-    # @example
-    #   list.each_word do |word|
-    #     puts word
-    #   end
-    #
-    def each_word(&block)
-    end
-
-    #
-    # Enumerates through every unique word in the list.
-    #
-    # @yield [word]
-    #   The given block will be passed each unique word in the list.
-    #
-    # @yieldparam [String] word
-    #   A unique word from the list.
-    #
-    # @example
-    #   list.each_unique do |word|
-    #     puts word
-    #   end
-    #
-    def each_unique
-      unique_filter = UniqueFilter.new()
-
-      each_word do |word|
-        if unique_filter.saw!(word)
-          yield word
-        end
-      end
-
-      unique_filter = nil
-    end
-
-    #
-    # Enumerates through every unique mutation, of every unique word, using
-    # the mutator rules define for the list.
-    #
-    # @yield [word]
-    #   The given block will be passed every mutation of every unique
-    #   word in the list.
-    #
-    # @yieldparam [String] word
-    #   A mutation of a unique word from the list.
-    #
-    # @example
-    #   list.each_mutation do |word|
-    #     puts word
-    #   end
-    #
-    def each_mutation(&block)
-      mutation_filter = UniqueFilter.new()
-
-      mutator_stack = [lambda { |mutated_word|
-        # skip words shorter than the minimum length
-        next if mutated_word.length < @min_length
-
-        # truncate words longer than the maximum length
-        mutated_word = mutated_word[0,@max_length] if @max_length
-
-        if mutation_filter.saw!(mutated_word)
-          yield mutated_word
-        end
-      }]
-
-      (@mutators.length-1).downto(0) do |index|
-        mutator_stack.unshift(lambda { |word|
-          prev_mutator = @mutators[index]
-          next_mutator = mutator_stack[index+1]
-
-          prev_mutator.each(word,&next_mutator)
-        })
-      end
-
-      each_unique(&(mutator_stack.first))
-    end
-
-    alias each each_mutation
-
-  end
-end

data/lib/wordlist/mutator.rb

@@ -1,113 +0,0 @@
-module Wordlist
-  class Mutator
-
-    include Enumerable
-
-    # The pattern to match
-    attr_accessor :pattern
-
-    # The data to substitute matched text with
-    attr_accessor :substitute
-
-    #
-    # Creates a new Mutator object.
-    #
-    # @param [String, Regexp] pattern
-    #   The pattern which recognizes text to mutate.
-    #
-    # @param [String, Integer] substitute
-    #   The optional text to replace recognized text.
-    #
-    # @yield [match]
-    #   If a block is given, it will be used to mutate recognized text.
-    #
-    # @yieldparam [String] match
-    #   The match text to mutate.
-    #
-    def initialize(pattern,substitute=nil,&block)
-      @pattern    = pattern
-      @substitute = (substitute || block)
-    end
-
-    #
-    # Mutates the given text.
-    #
-    # @param [String] matched
-    #   The recognized text to be mutated.
-    #
-    # @return [String]
-    #   The mutated text.
-    #
-    def replace(matched)
-      result = if @substitute.kind_of?(Proc)
-                 @substitute.call(matched)
-               else
-                 @substitute
-               end
-
-      result = if result.kind_of?(Integer)
-                 result.chr
-               else
-                 result.to_s
-               end
-
-      return result
-    end
-
-    #
-    # Enumerates over every possible mutation of the given word.
-    #
-    # @param [String] word
-    #   The word to mutate.
-    #
-    # @yield [mutation]
-    #   The given block will be passed every possible mutation of the
-    #   given word.
-    #
-    # @yieldparam [String] mutation
-    #   One possible mutation of the given word.
-    #
-    # @return [String]
-    #   The original word.
-    #
-    def each(word)
-      choices = 0
-
-      # first iteration
-      yield(word.gsub(@pattern) { |matched|
-        # determine how many possible choices there are
-        choices = ((choices << 1) | 0x1)
-
-        replace(matched)
-      })
-
-      (choices - 1).downto(0) do |iteration|
-        bits = iteration
-
-        yield(word.gsub(@pattern) { |matched|
-          result = if ((bits & 0x1) == 0x1)
-                     replace(matched)
-                   else
-                     matched
-                   end
-
-          bits >>= 1
-          result
-        })
-      end
-
-      return word
-    end
-
-    #
-    # Inspects the mutator.
-    #
-    # @return [String]
-    #   The inspected mutator.
-    #
-    def inspect
-      "#{@pattern.inspect} -> #{@substitute.inspect}"
-    end
-
-  end
-end

data/lib/wordlist/parsers.rb

@@ -1,74 +0,0 @@
-module Wordlist
-  module Parsers
-    # Ignore case of parsed text
-    attr_accessor :ignore_case
-
-    # Ignore the punctuation of parsed text
-    attr_accessor :ignore_punctuation
-
-    # Ignore URLs
-    attr_accessor :ignore_urls
-
-    # Ignore Phone numbers
-    attr_accessor :ignore_phone_numbers
-
-    # Ignore References
-    attr_accessor :ignore_references
-
-    #
-    # Initializes the parsers settings.
-    #
-    def initialize
-      @ignore_case          = false
-      @ignore_punctuation   = true
-      @ignore_urls          = true
-      @ignore_phone_numbers = false
-      @ignore_references    = false
-    end
-
-    #
-    # Parses the given text.
-    #
-    # @param [String] text
-    #   The text to parse.
-    #
-    # @return [Array<String>]
-    #   The Array of parsed tokens.
-    #
-    def parse(text)
-      text = text.to_s
-
-      if @ignore_punctuation
-        # eat tailing punctuation
-        text.gsub!(/[\.\?!]*$/,'')
-      end
-
-      if @ignore_case
-        # downcase the sentence
-        text.downcase!
-      end
-
-      if @ignore_urls
-        text.gsub!(/\s*\w+:\/\/[\w\/\+_\-,:%\d\.\-\?&=]*\s*/,' ')
-      end
-
-      if @ignore_phone_numbers
-        # remove phone numbers
-        text.gsub!(/\s*(\d-)?(\d{3}-)?\d{3}-\d{4}\s*/,' ')
-      end
-
-      if @ignore_references
-        # remove RFC style references
-        text.gsub!(/\s*[\(\{\[]\d+[\)\}\]]\s*/,' ')
-      end
-
-      if @ignore_punctuation
-        # split and ignore punctuation characters
-        return text.scan(/\w+[\-_\.:']\w+|\w+/)
-      else
-        # split and accept punctuation characters
-        return text.scan(/[\w\-_,:;\.\?\!'"\\\/]+/)
-      end
-    end
-  end
-end

data/lib/wordlist/runners/list.rb

@@ -1,116 +0,0 @@
-require 'wordlist/runners/runner'
-require 'wordlist/flat_file'
-
-module Wordlist
-  module Runners
-    class List < Runner
-
-      #
-      # Creates a new List Runner.
-      #
-      def initialize
-        @file = nil
-        @min_length = nil
-        @max_length = nil
-        @mutations = []
-
-        @words = false
-        @unique_words = false
-
-        @output = nil
-      end
-
-      #
-      # Runs the list runner.
-      #
-      # @param [Array<String>] args
-      #   Arguments to run the runner with.
-      #
-      def run(*args)
-        super(*args)
-
-        list = if @file
-                 FlatFile.new(
-                   @file,
-                   :min_length => @min_length,
-                   :max_length => @max_length
-                 )
-               else
-                 print_error('the --file option must be specified')
-                 exit -1
-               end
-
-        @mutations.each do |pattern,substitute|
-          list.mutate(pattern,substitute)
-        end
-
-        words = lambda { |output|
-          puts = output.method(:puts)
-
-          if @unique_words
-            list.each_unique(&puts)
-          elsif @words
-            list.each_word(&puts)
-          else
-            list.each(&puts)
-          end
-        }
-
-        if @output
-          File.open(@output,'w+',&words)
-        else
-          words.call(Kernel)
-        end
-      end
-
-      protected
-
-      #
-      # Parses the given arguments.
-      #
-      # @param [Array<String>] args
-      #   Arguments to parse.
-      #
-      def optparse(*args)
-        super(*args) do |opts|
-          opts.banner = 'usage: wordlist [options]'
-
-          opts.on('-f','--file FILE','The wordlist file to list') do |file|
-            @file = file
-          end
-
-          opts.on('--min-length NUM','Minimum length of words in characters') do |min|
-            @min_length = min
-          end
-
-          opts.on('--max-length NUM','Maximum length of words in characters') do |max|
-            @max_length = max
-          end
-
-          opts.on('-m','--mutate SUBSTRING::REPLACE','Adds a mutation rule') do |substring_and_replace|
-            @mutations << substring_and_replace.split('::',2)
-          end
-
-          opts.on('-M','--mutate-pattern PATTERN::REPLACE','Adds a mutation rule') do |pattern_and_replace|
-            pattern, replace = substring_and_replace.split('::',2)
-
-            @mutations << [Regexp.new(pattern), replace]
-          end
-
-          opts.on('-w','--words','Only print the words in the wordlist') do
-            @words = true
-          end
-
-          opts.on('-u','--unique','Only print the unique words in the wordlist') do
-            @unique_words = true
-          end
-
-          opts.on('-o','--output FILE','Optional file to output the wordlist to') do |file|
-            @output = File.expand_path(file)
-          end
-        end
-      end
-
-    end
-  end
-end

data/lib/wordlist/runners/runner.rb

@@ -1,67 +0,0 @@
-require 'optparse'
-
-module Wordlist
-  module Runners
-    class Runner
-      #
-      # Creates and runs the runner with the given arguments.
-      #
-      # @param [Array<String>] args
-      #   Arguments to parse.
-      #
-      def self.run(*args)
-        runner = new()
-        runner.run(*args)
-      end
-
-      #
-      # Runs the runner with the given arguments.
-      #
-      # @param [Array<String>] args
-      #   Arguments to run the runner with.
-      #
-      def run(*args)
-        optparse(*args)
-      end
-
-      protected
-
-      #
-      # Prints the given error message.
-      #
-      # @param [String] message
-      #   The error message to print.
-      #
-      def print_error(message)
-        $stderr.puts "#{$0}: #{message}"
-      end
-
-      #
-      # Parses the given arguments.
-      #
-      # @param [Array<String>] args
-      #   Arguments to parse.
-      #
-      # @yield [opts]
-      #   If a block is given, it will be passed the option parse to be
-      #   configured.
-      #
-      # @yieldparam [OptionParser] opts
-      #   The option parser to be configured.
-      #
-      def optparse(*args)
-        opts = OptionParser.new()
-
-        yield opts if block_given?
-
-        begin
-          opts.parse!(args)
-        rescue OptionParser::InvalidOption => e
-          $stderr.puts e.message
-          $stderr.puts opts
-          exit -1
-        end
-      end
-    end
-  end
-end

data/lib/wordlist/runners.rb

@@ -1,2 +0,0 @@
-require 'wordlist/runners/runner'
-require 'wordlist/runners/list'

data/scripts/benchmark

@@ -1,59 +0,0 @@
-#!/usr/bin/env ruby
-$LOAD_PATH.unshift(File.expand_path(File.join(File.dirname(__FILE__),'..','lib')))
-
-require 'wordlist'
-require 'benchmark'
-require 'fileutils'
-
-path = File.expand_path(File.join(File.dirname(__FILE__),'shakespeare_wordlist.txt'))
-
-FileUtils.rm_f(path)
-
-Benchmark.bm do |bm|
-  bm.report('build') do
-    Wordlist::Builder.build(path) do |wordlist|
-      wordlist.parse_file('/home/hal/shaks12.txt')
-    end
-  end
-
-  bm.report('each_unique') do
-    Wordlist::FlatFile.new(path) do |wordlist|
-      wordlist.each_unique { |word| word }
-    end
-  end
-
-  bm.report('each_mutation (1)') do
-    Wordlist::FlatFile.new(path) do |wordlist|
-      wordlist.mutate /o/i, '0'
-
-      wordlist.each_mutation { |word| word }
-    end
-  end
-
-  bm.report('each_mutation (2)') do
-    Wordlist::FlatFile.new(path) do |wordlist|
-      wordlist.mutate /o/i, '0'
-      wordlist.mutate /a/i, '@'
-
-      wordlist.each_mutation { |word| word }
-    end
-  end
-
-  bm.report('each_mutation (3)') do
-    Wordlist::FlatFile.new(path) do |wordlist|
-      wordlist.mutate /o/i, '0'
-      wordlist.mutate /a/i, '@'
-      wordlist.mutate /e/i, '3'
-
-      wordlist.each_mutation { |word| word }
-    end
-  end
-end
-
-Benchmark.bm do |bm|
-  mutator = Wordlist::Mutator.new(/o/i, '0')
-
-  bm.report('Mutator#each') do
-    mutator.each('lololololoLOLOLOLOLO') { |word| }
-  end
-end