wordlist 0.1.0

data/spec/builder_spec.rb

@@ -0,0 +1,36 @@
+require 'wordlist/builder'
+
+require 'spec_helper'
+require 'helpers/text'
+require 'builder_examples'
+
+require 'tempfile'
+require 'fileutils'
+
+describe Builder do
+  describe "new wordlist" do
+    before(:all) do
+      @expected = ['dog', 'cat', 'catx', 'dat']
+    end
+
+    before(:each) do
+      @path = Tempfile.new('wordlist').path
+    end
+
+    it_should_behave_like "a wordlist Builder"
+  end
+
+  describe "existing wordlist" do
+    before(:all) do
+      @path = '/tmp/bla'
+      @expected = ['dog', 'cat', 'log', 'catx', 'dat']
+    end
+
+    before(:each) do
+      @path = Tempfile.new('wordlist').path
+      FileUtils.cp(PREVIOUS_WORDLIST,@path)
+    end
+
+    it_should_behave_like "a wordlist Builder"
+  end
+end

data/spec/classes/parser_class.rb

@@ -0,0 +1,5 @@
+class ParserClass
+
+  include Wordlist::Parsers
+
+end

data/spec/classes/test_list.rb

@@ -0,0 +1,9 @@
+require 'wordlist/list'
+
+class TestList < Wordlist::List
+
+  def each_word
+    yield 'omg.hackers'
+  end
+
+end

data/spec/flat_file_spec.rb

@@ -0,0 +1,25 @@
+require 'wordlist/flat_file'
+
+require 'spec_helper'
+
+describe FlatFile do
+  before(:all) do
+    @path = File.join(File.dirname(__FILE__),'text','flat_file.txt')
+    @list = FlatFile.new(@path)
+  end
+
+  it "should have a path it reads from" do
+    @list.path.should == @path
+  end
+
+  it "should read the lines of the flat-file" do
+    words = ['one', 'two', 'three']
+
+    @list.each_word do |word|
+      words.include?(word).should == true
+      words.delete(word)
+    end
+
+    words.should == []
+  end
+end

data/spec/list_spec.rb

@@ -0,0 +1,58 @@
+require 'wordlist/list'
+
+require 'spec_helper'
+require 'classes/test_list'
+
+describe List do
+  before(:all) do
+    @source = TestList.new
+    @source.mutate 'o', '0'
+    @source.mutate 'a', 'A'
+    @source.mutate 'e', '3'
+    @source.mutate 's', '5'
+  end
+
+  it "should iterate over each word" do
+    words = []
+
+    @source.each_word { |word| words << word }
+
+    words.should == ['omg.hackers']
+  end
+
+  it "should iterate over each unique word" do
+    words = []
+
+    @source.each_unique { |word| words << word }
+
+    words.should == ['omg.hackers']
+  end
+
+  it "should iterate over every possible mutated word" do
+    mutations = %w{
+      0mg.hAck3r5
+      0mg.hAck3rs
+      0mg.hAcker5
+      0mg.hAckers
+      0mg.hack3r5
+      0mg.hack3rs
+      0mg.hacker5
+      0mg.hackers
+      omg.hAck3r5
+      omg.hAck3rs
+      omg.hAcker5
+      omg.hAckers
+      omg.hack3r5
+      omg.hack3rs
+      omg.hacker5
+      omg.hackers
+    }
+
+    @source.each_mutation do |mutation|
+      mutations.include?(mutation).should == true
+      mutations.delete(mutation)
+    end
+
+    mutations.should == []
+  end
+end

data/spec/mutator_spec.rb

@@ -0,0 +1,43 @@
+require 'wordlist/mutator'
+
+require 'spec_helper'
+
+describe Mutator do
+  it "should replace matched text with a byte" do
+    mutator = Mutator.new('o',0x41)
+    mutator.replace('o').should == 'A'
+  end
+
+  it "should replace matched text with a String" do
+    mutator = Mutator.new('o','0')
+    mutator.replace('o').should == '0'
+  end
+
+  it "should replace matched text using a proc" do
+    mutator = Mutator.new('o') { |match| match * 2 }
+    mutator.replace('o').should == 'oo'
+  end
+
+  it "should iterate over every possible substitution" do
+    remaining = ['lolol', 'l0lol', 'lol0l', 'l0l0l']
+
+    mutator = Mutator.new(/o/,'0')
+    mutator.each('lolol') do |mutation|
+      remaining.include?(mutation).should == true
+      remaining.delete(mutation)
+    end
+
+    remaining.should == []
+  end
+
+  it "should iterate over the original word, if no matches were found" do
+    mutations = []
+    mutator = Mutator.new('x','0')
+
+    mutator.each('hello') do |mutant|
+      mutations << mutant
+    end
+
+    mutations.should == ['hello']
+  end
+end

data/spec/parsers_spec.rb

@@ -0,0 +1,118 @@
+require 'spec_helper'
+require 'classes/parser_class'
+
+describe Parsers do
+  describe "default" do
+    before(:all) do
+      @parser = ParserClass.new
+    end
+
+    it "should parse words from a sentence" do
+      sentence = %{The Deliverator is in touch with the road, starts like a bad day, stops on a peseta.}
+      words = %w{The Deliverator is in touch with the road starts like a bad day stops on a peseta}
+
+      @parser.parse(sentence).should == words
+    end
+
+    it "should ignore punctuation by default while parsing a sentence" do
+      sentence = %{Oh, they used to argue over times, many corporate driver-years lost to it: homeowners, red-faced and sweaty with their own lies, stinking of Old Spice and job-related stress, standing in their glowing yellow doorways brandishing their Seikos and waving at the clock over the kitchen sink, I swear, can't you guys tell time?}
+      words = %w{
+      Oh they used to argue over times many corporate driver-years lost to it homeowners red-faced and sweaty with their own lies stinking of Old Spice and job-related stress standing in their glowing yellow doorways brandishing their Seikos and waving at the clock over the kitchen sink I swear can't you guys tell time
+      }
+
+      @parser.parse(sentence).should == words
+    end
+
+    it "should ignore URLs by default while parsing a sentence" do
+      sentence = %{Click on the following link: http://www.example.com/}
+      words = %w{Click on the following link}
+
+      @parser.parse(sentence).should == words
+    end
+
+    it "should ignore short URIs by default while parsing a sentence" do
+      sentence = %{Click on the following link: jabber://}
+      words = %w{Click on the following link}
+
+      @parser.parse(sentence).should == words
+    end
+
+    it "should ignore complex HTTP URLs by default while parsing a sentence" do
+      sentence = %{Click on the following link: http://www.google.com/search?hl=en&client=firefox-a&rls=org.mozilla:en-US:official&hs=jU&q=ruby+datamapper&start=20&sa=N}
+      words = %w{Click on the following link}
+
+      @parser.parse(sentence).should == words
+    end
+  end
+
+  describe "ignoring phone numbers" do
+    before(:all) do
+      @parser = ParserClass.new
+      @parser.ignore_phone_numbers = true
+    end
+
+    it "may ignore phone numbers while parsing a sentence" do
+      sentence = %{Call me before 12, 1-888-444-2222.}
+      words = %w{Call me before 12}
+
+      @parser.parse(sentence).should == words
+    end
+
+    it "may ignore long-distance phone numbers while parsing a sentence" do
+      sentence = %{Call me before 12, 1-444-2222.}
+      words = %w{Call me before 12}
+
+      @parser.parse(sentence).should == words
+    end
+
+    it "may ignore short phone numbers while parsing a sentence" do
+      sentence = %{Call me before 12, 444-2222.}
+      words = %w{Call me before 12}
+
+      @parser.parse(sentence).should == words
+    end
+  end
+
+  describe "ignoring references" do
+    before(:all) do
+      @parser = ParserClass.new
+      @parser.ignore_references = true
+    end
+
+    it "may ignore RFC style references while parsing a sentence" do
+      sentence = %{As one can see, it has failed [1].}
+      words = %w{As one can see it has failed}
+
+      @parser.parse(sentence).should == words
+    end
+  end
+
+  describe "ignoring case" do
+    before(:all) do
+      @parser = ParserClass.new
+      @parser.ignore_case = true
+    end
+
+    it "may ignore case while parsing a sentence" do
+      sentence = %{The Deliverator is in touch with the road, starts like a bad day, stops on a peseta.}
+      words = %w{the deliverator is in touch with the road starts like a bad day stops on a peseta}
+
+      @parser.ignore_case = true
+      @parser.parse(sentence).should == words
+    end
+  end
+
+  describe "preserving punctuation" do
+    before(:all) do
+      @parser = ParserClass.new
+      @parser.ignore_punctuation = false
+    end
+
+    it "may preserve punctuation while parsing a sentence" do
+      sentence = %{Oh, they used to argue over times, many corporate driver-years lost to it: homeowners, red-faced and sweaty with their own lies, stinking of Old Spice and job-related stress, standing in their glowing yellow doorways brandishing their Seikos and waving at the clock over the kitchen sink, I swear, can't you guys tell time?}
+      words = %w{Oh, they used to argue over times, many corporate driver-years lost to it: homeowners, red-faced and sweaty with their own lies, stinking of Old Spice and job-related stress, standing in their glowing yellow doorways brandishing their Seikos and waving at the clock over the kitchen sink, I swear, can't you guys tell time?}
+
+      @parser.parse(sentence).should == words
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,7 @@
+require 'rubygems'
+gem 'rspec', '>=1.1.12'
+require 'spec'
+
+require 'wordlist/version'
+
+include Wordlist

data/spec/text/flat_file.txt

@@ -0,0 +1,3 @@
+one
+two
+three

data/spec/text/previous_wordlist.txt

@@ -0,0 +1,3 @@
+cat
+dog
+log

data/spec/text/sample.txt

@@ -0,0 +1,3 @@
+dog cat, cat dat catx.
+dog, dog, dog,
+cat. dog, dat.

data/spec/unique_filter_spec.rb

@@ -0,0 +1,34 @@
+require 'wordlist/unique_filter'
+
+require 'spec_helper'
+
+describe UniqueFilter do
+  before(:each) do
+    @filter = UniqueFilter.new
+  end
+
+  it "should have seen words" do
+    @filter.saw!('cat')
+
+    @filter.seen?('cat').should == true
+    @filter.seen?('dog').should == false
+  end
+
+  it "should only see a unique word once" do
+    @filter.saw!('cat').should == true
+    @filter.saw!('cat').should == false
+  end
+
+  it "should pass only unique words through the filter" do
+    input = ['dog', 'cat', 'dog']
+    output = []
+
+    input.each do |word|
+      @filter.pass(word) do |result|
+        output << result
+      end
+    end
+
+    output.should == ['dog', 'cat']
+  end
+end

data/spec/wordlist_spec.rb

@@ -0,0 +1,9 @@
+require 'wordlist/version'
+
+require 'spec_helper'
+
+describe Wordlist do
+  it "should have a VERSION constant" do
+    Wordlist.const_defined?('VERSION').should == true
+  end
+end

data/tasks/spec.rb

@@ -0,0 +1,9 @@
+require 'spec/rake/spectask'
+
+desc "Run all specifications"
+Spec::Rake::SpecTask.new(:spec) do |t|
+  t.libs += ['lib', 'spec']
+  t.spec_opts = ['--colour', '--format', 'specdoc']
+end
+
+task :default => :spec

metadata

@@ -0,0 +1,123 @@
+--- !ruby/object:Gem::Specification 
+name: wordlist
+version: !ruby/object:Gem::Version 
+  version: 0.1.0
+platform: ruby
+authors: 
+- Postmodern
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-08-31 00:00:00 -07:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: spidr
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 0.1.9
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.1.12
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: hoe
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 2.3.3
+    version: 
+description: |-
+  A Ruby library for generating and working with word-lists. Wordlist allows
+  one to efficiently generate unique word-lists from arbitrary text or
+  other sources, such as website content. Wordlist can also quickly enumerate
+  through words within an existing word-list, applying multiple mutation
+  rules to each word in the list.
+email: 
+- postmodern.mod3@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- History.txt
+- Manifest.txt
+- README.txt
+- scripts/text/comedy_of_errors.txt
+files: 
+- History.txt
+- Manifest.txt
+- README.txt
+- Rakefile
+- lib/wordlist.rb
+- lib/wordlist/unique_filter.rb
+- lib/wordlist/parsers.rb
+- lib/wordlist/builder.rb
+- lib/wordlist/builders.rb
+- lib/wordlist/builders/website.rb
+- lib/wordlist/mutator.rb
+- lib/wordlist/list.rb
+- lib/wordlist/flat_file.rb
+- lib/wordlist/version.rb
+- tasks/spec.rb
+- scripts/benchmark
+- scripts/text/comedy_of_errors.txt
+- spec/classes/parser_class.rb
+- spec/classes/test_list.rb
+- spec/text/previous_wordlist.txt
+- spec/text/sample.txt
+- spec/text/flat_file.txt
+- spec/spec_helper.rb
+- spec/unique_filter_spec.rb
+- spec/parsers_spec.rb
+- spec/mutator_spec.rb
+- spec/builder_spec.rb
+- spec/list_spec.rb
+- spec/flat_file_spec.rb
+- spec/wordlist_spec.rb
+has_rdoc: true
+homepage: http://wordlist.rubyforge.org/
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --main
+- README.txt
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: wordlist
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 3
+summary: A Ruby library for generating and working with word-lists
+test_files: []
+