nlp_backpack 0.0.0

data/.document

@@ -0,0 +1,5 @@
+README.rdoc
+lib/**/*.rb
+bin/*
+features/**/*.feature
+LICENSE

data/.gitignore

@@ -0,0 +1,21 @@
+## MAC OS
+.DS_Store
+
+## TEXTMATE
+*.tmproj
+tmtags
+
+## EMACS
+*~
+\#*
+.\#*
+
+## VIM
+*.swp
+
+## PROJECT::GENERAL
+coverage
+rdoc
+pkg
+
+## PROJECT::SPECIFIC

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 reddavis
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,22 @@
+= NLP Backpack
+
+An NLTK like playground for Rubyists. A notebook of my studies
+
+If a fish is a fish, what is a fish?
+
+== TODO
+
+* Lots
+
+Classifiers
+* Lots of them!
+
+Chunkers
+* IOB tags
+* Tree structure
+* Unigram Chunker
+
+Data
+
+Evaluations
+* F Score

data/Rakefile

@@ -0,0 +1,45 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "nlp_backpack"
+    gem.summary = %Q{A backpack full of useful toys}
+    gem.description = %Q{A backpack full of useful toys}
+    gem.email = "reddavis@gmail.com"
+    gem.homepage = "http://github.com/reddavis/NLP-Backpack"
+    gem.authors = ["reddavis"]
+    gem.add_development_dependency "rspec", ">= 1.2.9"
+    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
+end
+
+require 'spec/rake/spectask'
+Spec::Rake::SpecTask.new(:spec) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.spec_files = FileList['spec/**/*_spec.rb']
+end
+
+Spec::Rake::SpecTask.new(:rcov) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.pattern = 'spec/**/*_spec.rb'
+  spec.rcov = true
+end
+
+task :spec => :check_dependencies
+
+task :default => :spec
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "nlp_backpack #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+0.0.0

data/lib/nlp_backpack.rb

@@ -0,0 +1,10 @@
+$:.unshift File.expand_path(File.dirname(__FILE__))
+
+module NLPBackpack
+  autoload :FrequencyDistribution, "nlp_backpack/frequency_distribution"
+  autoload :Evaluation, "nlp_backpack/evaluation"
+  autoload :Classifier, "nlp_backpack/classifier"
+  autoload :StopWords, "nlp_backpack/stop_words"
+  autoload :Chunker, "nlp_backpack/chunker"
+  autoload :POS, "nlp_backpack/pos"
+end

data/lib/nlp_backpack/chunker.rb

@@ -0,0 +1,5 @@
+module NLPBackpack
+  module Chunker
+    autoload :RegexChunker, "nlp_backpack/chunker/regex_chunker"
+  end
+end

data/lib/nlp_backpack/chunker/regex_chunker.rb

@@ -0,0 +1,107 @@
+# ChunkGrammer
+# Chunk Grammers are regex-esq patterns that indicate how sentences should be chunked
+# The patterns are made up of Tag Patterns.
+# (DT)?(JJ.*)* == Optional determiner, followed by zero or more of any type of adjective.
+
+require 'nlp_backpack/chunker/tag_pattern'
+
+module NLPBackpack
+
+  module Chunker
+    class Retry < Exception; end;
+
+    class RegexChunker
+
+      def initialize(pattern)
+        @pattern = prepare_pattern(pattern)
+
+        @cached_pattern = @pattern.clone
+        @cached_pattern.freeze
+
+        @matched_patterns = []
+        @potential_pattern = []
+      end
+
+      # Extract all matches
+      def match(pos_array)
+        next_pattern(:start)
+
+        pos_array.each do |word|
+          begin
+            pos = word[1]
+
+            if pos.match(@current_pattern.tag)
+              @potential_pattern << word[0]
+              next_pattern(:matched)
+            else
+              next_pattern(:no_match)
+            end
+          rescue Retry
+            retry
+          end
+        end
+
+        pop_potential_pattern!
+
+        @matched_patterns.map {|pattern| pattern.join(" ") }
+      end
+
+      private
+
+      def next_pattern(state)
+        if @pattern.empty?
+          reset_pattern!
+          next_pattern(:start)
+        end
+
+        @current_pattern = case state
+        when :start, :next
+          @pattern.pop
+        when :matched
+          if ["+", "*"].include?(@current_pattern.conditions)
+            @current_pattern
+          else
+            @pattern.pop
+          end
+        when :no_match
+          if ["?", "*"].include?(@current_pattern.conditions)
+            next_pattern(:next)
+            raise Retry #Prob not the best way to do this(?)
+          else
+            # Start again
+            reset_pattern!
+            pop_potential_pattern!
+            next_pattern(:start)
+          end
+        end
+      end
+
+      def pop_potential_pattern!
+        if @potential_pattern.size >= minimum_potential_pattern_size
+          @matched_patterns << @potential_pattern
+        end
+
+        reset_potential_pattern!
+      end
+
+      def reset_pattern!
+        @pattern = @cached_pattern.dup
+      end
+
+      def reset_potential_pattern!
+        @potential_pattern = []
+      end
+
+      def minimum_potential_pattern_size
+        @cached_pattern.reject {|x| ["?", "*"].include?(x.conditions)}.size
+      end
+
+      # Extract patterns like (DT)? -- (JJ.*)*
+      def prepare_pattern(pattern)
+        pattern.scan(/(\<[^\>]+\>[^\<]?)/).flatten.map { |x| TagPattern.new(x) }.reverse
+      end
+
+    end
+  end
+
+end

data/lib/nlp_backpack/chunker/tag_pattern.rb

@@ -0,0 +1,31 @@
+module NLPBackpack
+
+  module Chunker
+    class TagPattern
+
+      attr_reader :tag, :conditions
+
+      # Example inputs:
+      # <DT>?
+      # <JJ.*>*
+      def initialize(pattern)
+        extract_tag_and_options(pattern)
+      end
+
+      private
+
+      # TODO Make this work for strings wrapped in " " as well as ' '
+      def extract_tag_and_options(pattern)
+        if match = pattern.match(/\<([^\>]+)\>/)
+          @tag = /#{match[1]}/
+        end
+
+        if match = pattern.match(/\<[^\>]+\>(.)/)
+          @conditions = match[1]
+        end
+      end
+
+    end
+  end
+
+end

data/lib/nlp_backpack/classifier.rb

@@ -0,0 +1,5 @@
+module NLPBackpack
+  module Classifier
+    autoload :NaiveBayes, "nlp_backpack/classifier/naive_bayes"
+  end
+end

data/lib/nlp_backpack/classifier/base.rb

@@ -0,0 +1,28 @@
+module NLPBackpack
+  module Classifier
+
+    class Base
+      class << self
+        def load(db_path)
+          data = ""
+          File.open(db_path) do |f|
+            while line = f.gets
+              data << line
+            end
+          end
+          Marshal.load(data)
+        end
+      end
+
+      attr_accessor :db_filepath
+
+      def save
+        raise "You haven't set a db_filpath, I dont know where to save" if @db_filepath.nil?
+        File.open(@db_filepath, "w+") do |f|
+          f.write(Marshal.dump(self))
+        end
+      end
+    end
+
+  end
+end

data/lib/nlp_backpack/classifier/naive_bayes.rb

@@ -0,0 +1,83 @@
+# Bayes Theorem
+# P(A|B) = P(B|A) * P(A) / P(B)
+
+# Terminology
+# An ITEM is made up of FEATURES
+# An ITEM belongs to a CLASS
+
+# Bayes With Our Terminology
+# P(Class | Item) = P(Item | Class) * P(Class) / P(Item)
+
+# However, when classifying, P(Item) is the same across all calcualtions
+# So we don't bother to calculate it
+
+require "nlp_backpack/classifier/base"
+
+module NLPBackpack
+  module Classifier
+
+    class NaiveBayes < Base
+      def initialize(*klasses)
+        @features_count = {}
+        @klass_count = {}
+        @klasses = klasses
+
+        klasses.each do |klass|
+          @features_count[klass] = Hash.new(0.0)
+          @klass_count[klass] = 0.0
+        end
+      end
+
+      def train(klass, *features)
+        features.uniq.each do |feature|
+          @features_count[klass][feature] += 1
+        end
+        @klass_count[klass] += 1
+      end
+
+      #P(Class | Item) = P(Item | Class) * P(Class)
+      def classify(*features)
+        scores = {}
+        @klasses.each do |klass|
+          scores[klass] = (prob_of_item_given_a_class(features, klass) * prob_of_class(klass))
+        end
+        scores.sort {|a,b| b[1] <=> a[1]}[0]
+      end
+
+      private
+
+      # P(Item | Class)
+      def prob_of_item_given_a_class(features, klass)
+        a = features.inject(1.0) do |sum, feature|
+          prob = prob_of_feature_given_a_class(feature, klass)
+          sum *= prob
+        end
+      end
+
+      # P(Feature | Class)
+      def prob_of_feature_given_a_class(feature, klass)
+        return assumed_probability if @features_count[klass][feature] == 0
+        @features_count[klass][feature] / @klass_count[klass]
+      end
+
+      # P(Class)
+      def prob_of_class(klass)
+        @klass_count[klass] / total_items
+      end
+
+      def total_items
+        @klass_count.inject(0) do |sum, klass|
+          sum += klass[1]
+        end
+      end
+
+      # If we have only trained a little bit a class may not have had a feature yet
+      # give it a probability of 0 may not be true so we produce a assumed probability
+      # which gets smaller more we train
+      def assumed_probability
+        0.5 / (total_items/2)
+      end
+    end
+
+  end
+end

data/lib/nlp_backpack/evaluation.rb

@@ -0,0 +1,6 @@
+module NLPBackpack
+  module Evaluation
+    autoload :Accuracy, "nlp_backpack/evaluation/accuracy"
+    autoload :ConfusionMatrix, "nlp_backpack/evaluation/confusion_matrix"
+  end
+end

data/lib/nlp_backpack/evaluation/accuracy.rb

@@ -0,0 +1,46 @@
+require 'nlp_backpack/evaluation/base'
+
+module NLPBackpack
+  module Evaluation
+
+    class Accuracy < Base
+
+      def accuracy_of(klass)
+        results[klass]
+      end
+
+      def inspect
+        output = ""
+
+        results.each do |klass, result|
+          output << "#{klass}: #{result}% correct\n"
+        end
+
+        output
+      end
+
+      private
+
+      def results
+        @results ||= begin
+          correct_klass_count = Hash.new {|h,k| h[k] = 0.0}
+          total_klass_count = Hash.new {|h,k| h[k] = 0.0}
+
+          @correct_results.each_with_index do |correct_result, index|
+            total_klass_count[correct_result] += 1
+            correct_klass_count[correct_result] += 1 if correct_result == @test_results[index]
+          end
+
+          results = Hash.new
+
+          total_klass_count.each do |klass, total_count|
+            results[klass] = (correct_klass_count[klass] / total_count * 100).round
+          end
+
+          results
+        end
+      end
+    end
+
+  end
+end