word_stats 0.0.1

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.rspec

@@ -0,0 +1 @@
+--color

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in word_stats.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Casper
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,61 @@
+# WordStats
+
+WordStats provides a set of methods useful for counting character and word frequencies. 
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'word_stats'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install word_stats
+
+## Usage
+
+Require the WordStats gem as follows:
+
+    require 'word_stats' # Remember to require Ruby Gems first if using Ruby 1.8
+
+    text = "The quick brown fox jumps over the lazy dog."
+    # Note: all strings processed by WordStats are downcased!!
+
+WordStats provides shortcuts for single letter frequencies, bigrams and trigrams. The `WordStats::Characters.ngrams(n,text)` method can be used to find n-grams of any length. The output is a hash of the form [:word,count]. 
+
+    letter_frequencies = WordStats::Characters.letters(text)
+    letter_frequencies[:'u'] #=> 2
+    
+    bigrams = WordStats::Characters.bigrams(text)
+    bigrams[:'th'] #=> 2
+    
+    trigrams = WordStats::Characters.trigrams(text)
+    trigrams['qui'.to_sym] #=> 1
+    
+    octocats = WordStats::Characters.ngrams(8,text)
+    octocats[:'The quic'] #=> 0
+    octocats[:'the quic'] #=> 1
+
+Similarly, WordStats provides a method to count words and any arbitrary length sequence of words:
+
+    word_count = WordStats::Words.nwords(1,text)
+    word_count[:'the'] #=> 2
+
+    word_pairs = WordStats::Words.nwords(2,text)
+    word_pairs[:'quick brown'] #=> 1
+
+## Important Notes
+
+WordStats will downcase any string that you pass into it. It also strips punctuation before processing.
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Added some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,2 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"

data/lib/word_stats/string_extensions.rb

@@ -0,0 +1,8 @@
+class String
+  ##
+  # Removes all common forms of punctuation marks from the string.
+  def remove_punctuation
+    regex = /[\.\,\!\?\;\[\]\(\)\'\"\/\\\`]/
+    self.gsub(regex,'')
+  end
+end

data/lib/word_stats/version.rb

@@ -0,0 +1,3 @@
+module WordStats
+  VERSION = "0.0.1"
+end

data/lib/word_stats.rb

@@ -0,0 +1,53 @@
+require "word_stats/version"
+require "word_stats/string_extensions"
+
+##
+# This module exposes a collection of methods available for statistical analysis of strings of text. 
+module WordStats
+
+  module Characters
+
+    ##
+    # Computes a letter frequency distribution of the input string
+    def self.letters(text)
+      ngrams(1,text)
+    end
+
+    ## 
+    # Computes the bigram frequency of the input string
+    def self.bigrams(text)
+      ngrams(2,text)
+    end
+
+    ##
+    # Computes the trigram frequency of the input string
+    def self.trigrams(text)
+      ngrams(3,text)
+    end
+
+    ##
+    # Compute the N-gram frequency of the input string
+    def self.ngrams(n,text)
+      text = text.remove_punctuation.downcase
+      ngrams = Hash.new(0)
+      for i in n-1..(text.length-1)
+        ngrams[(text[i-(n-1)..i]).to_sym] += 1
+      end
+      ngrams
+    end
+
+  end
+
+  module Words
+    def self.nwords(n,text)
+      text = text.remove_punctuation.downcase
+      text = text.split(/\s+/)
+      nwords = Hash.new(0)
+      for i in n-1..(text.length-1)
+        nwords[(text[i-(n-1)..i]).join(' ').to_sym] += 1
+      end
+      nwords
+    end
+  end
+
+end

data/spec/characters_spec.rb

@@ -0,0 +1,36 @@
+require 'word_stats'
+
+describe WordStats::Characters do
+  it "should produce a correct letter distribution" do
+    text = "abbcccdddd"
+    letters = WordStats::Characters.ngrams(1,text)
+    letters[:'d'].should == 4
+    letters[:'b'].should == 2
+    letters[:'e'].should == 0
+  end
+
+  it "should correctly identify the bigrams" do
+    text = "The quick brown fox jumps over the lazy dog."
+    bigrams = WordStats::Characters.bigrams(text)
+    bigrams[:'th'].should == 2
+    bigrams[:'he'].should == 2
+    bigrams[:'qu'].should == 1
+    bigrams[:'g.'].should == 0
+  end
+
+  it "should correctly identify trigrams" do
+    text = "Harharhar! It was just a harmless joke!"
+    trigrams = WordStats::Characters.trigrams(text)
+    trigrams[:'har'].should == 4
+    trigrams[:'s j'].should == 2
+    trigrams[:'ke!'].should == 0
+  end
+
+  it "should correctly identify 8-grams" do
+    text = "The octopus has eight legs."
+    octograms = WordStats::Characters.ngrams(8,text)
+    octograms[:' octopus'].should == 1
+    octograms[:'has nine'].should == 0
+  end
+
+end

data/spec/string_spec.rb

@@ -0,0 +1,26 @@
+require 'word_stats'
+
+describe String,"#remove_punctuation" do
+  it "should remove exclamation points, periods, and question marks" do
+    text = "Hello! My name is Casper. How may I help you?"
+    text.remove_punctuation.should == "Hello My name is Casper How may I help you"
+  end
+  it "should remove single quotes and double quotes" do
+    text = "\'Hey,\' said the \"Dude\"."
+    text.remove_punctuation.should == "Hey said the Dude"
+  end
+  it "should remove parenthesis and brackets" do
+    text = "fo[ob].ar()"
+    text.remove_punctuation.should == "foobar"
+  end
+  it "should remove backticks and semicolons" do
+    text = "Java stmts end in semicolons; Ruby does cool things like `ls`."
+    text.remove_punctuation.should == "Java stmts end in semicolons Ruby does cool things like ls"
+  end
+  it "should remove dashes if set to do so" do
+    pending "configuration preferences or hash object passed to method?"
+  end
+  it "should remove underscores if set to do so" do
+    pending "same as dashes"
+  end
+end

data/spec/words_spec.rb

@@ -0,0 +1,16 @@
+require 'word_stats'
+
+describe WordStats::Words do
+  it "should correctly count words" do
+    text = "How many times have I used the word times?"
+    words = WordStats::Words.nwords(1,text)
+    words[:'times'].should == 2
+    words[:'have'].should == 1
+  end
+  it "should correctly identify word pairs" do
+    text = "Sometimes I run. Sometimes I jump. Sometimes, however, I sit."
+    words = WordStats::Words.nwords(2,text)
+    words[:'sometimes i'].should == 2
+    words[:'never do'].should == 0
+  end
+end

data/word_stats.gemspec

@@ -0,0 +1,17 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/word_stats/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Casper"]
+  gem.email         = ["cholmgreen@gmail.com"]
+  gem.description   = "Provides a set of methods useful for analysis of plaintext"
+  gem.summary       = "Provides a set of methods useful for analysis of plaintext"
+  gem.homepage      = ""
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "word_stats"
+  gem.require_paths = ["lib"]
+  gem.version       = WordStats::VERSION
+end

metadata

@@ -0,0 +1,61 @@
+--- !ruby/object:Gem::Specification
+name: word_stats
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Casper
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-08-06 00:00:00.000000000 Z
+dependencies: []
+description: Provides a set of methods useful for analysis of plaintext
+email:
+- cholmgreen@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rspec
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- lib/word_stats.rb
+- lib/word_stats/string_extensions.rb
+- lib/word_stats/version.rb
+- spec/characters_spec.rb
+- spec/string_spec.rb
+- spec/words_spec.rb
+- word_stats.gemspec
+homepage: ''
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: Provides a set of methods useful for analysis of plaintext
+test_files:
+- spec/characters_spec.rb
+- spec/string_spec.rb
+- spec/words_spec.rb