josephwilk-rsemantic 0.1.0

data/History.txt

@@ -0,0 +1,4 @@
+== 0.0.1 
+
+* Changed internal representation of vector space. Using columns as documents and rows as terms. This is more consistent which LSA research papers.
+* Wrap DMatrix in VectorSpace::Model, allowing us to store keywords with the matrix and get pretty output

data/Manifest.txt

@@ -0,0 +1,38 @@
+History.txt
+Manifest.txt
+README.txt
+Rakefile
+TODO.txt
+config/hoe.rb
+config/requirements.rb
+gem_tasks/deployment.rake
+gem_tasks/environment.rake
+gem_tasks/examples.rake
+gem_tasks/fix_cr_lf.rake
+gem_tasks/gemspec.rake
+gem_tasks/rspec.rake
+gem_tasks/website.rake
+lib/semantic.rb
+lib/semantic/compare.rb
+lib/semantic/matrix_transformer.rb
+lib/semantic/parser.rb
+lib/semantic/search.rb
+lib/semantic/transform.rb
+lib/semantic/transform/lsa_transform.rb
+lib/semantic/transform/tf_idf_transform.rb
+lib/semantic/vector_space.rb
+lib/semantic/vector_space/builder.rb
+lib/semantic/vector_space/model.rb
+lib/semantic/version.rb
+resources/english.stop
+rsemantic.gemspec
+spec/semantic/compare_spec.rb
+spec/semantic/matrix_transformer_spec.rb
+spec/semantic/parser_spec.rb
+spec/semantic/search_spec.rb
+spec/semantic/transform/lsa_transform_spec.rb
+spec/semantic/transform/tf_idf_transform_spec.rb
+spec/semantic/vector_space/builder_spec.rb
+spec/semantic/vector_space/model_spec.rb
+spec/spec.opts
+spec/spec_helper.rb

data/README.txt

@@ -0,0 +1,48 @@
+= Rsemantic
+
+* http://github.com/josephwilk/rsemantic
+
+== DESCRIPTION:
+
+A Ruby document vector search with flexible matrix transforms. Current supported transforms: 
+
+* Latent semantic analysis
+* Term frequency - inverse document frequency
+
+Documentation: http://github.com/josephwilk/rsemantic/wikis/home
+
+== REQUIREMENTS:
+
+* Linalg - http://rubyforge.org/projects/linalg/
+* stemmer - http://rubyforge.org/projects/stemmer/
+
+== INSTALL:
+
+* git clone git://github.com/josephwilk/rsemantic.git
+
+== LICENSE
+
+(The MIT License)
+
+Copyright (c) 2008 Joseph Wilk
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+

data/Rakefile

@@ -0,0 +1,9 @@
+ENV['NODOT'] = 'true' # We don't want class diagrams in RDoc
+require 'config/requirements'
+require 'config/hoe' # setup Hoe + all gem configuration
+
+Dir['gem_tasks/**/*.rake'].each { |rake| load rake }
+
+# Hoe gives us :default => :test, but we don't have Test::Unit tests.
+Rake::Task[:default].clear_prerequisites
+task :default => [:spec]

data/TODO.txt

@@ -0,0 +1,9 @@
+== FEATURES/PROBLEMS:
+
+* Applying transforms to query vectors
+* Detect the optimal dimension reduction in LSA.
+* Allow objects to be passed in as transforms.
+* Implement Probabilistic latent semantic analysis
+* Implement Latent Dirichlet Allocation
+
+* Matrix transformer has to popout the matrix of VectorSpace::Model and reassign it, get rid of this.

data/config/hoe.rb

@@ -0,0 +1,69 @@
+require 'semantic/version'
+
+AUTHOR = 'Joseph Wilk'  # can also be an array of Authors
+EMAIL = "josephwilk@joesniff.co.uk"
+DESCRIPTION = "A document vector search with flexible matrix transforms. Currently supports Latent semantic analysis and Term frequency - inverse document frequency"
+GEM_NAME = 'rsemantic' # what ppl will type to install your gem
+HOMEPATH = "http://github.com/josephwilk/rsemantic"
+RUBYFORGE_PROJECT = 'rsemantic'
+
+@config_file = "~/.rubyforge/user-config.yml"
+@config = nil
+RUBYFORGE_USERNAME = "joseph_wilk"
+def rubyforge_username
+  unless @config
+    begin
+      @config = YAML.load(File.read(File.expand_path(@config_file)))
+    rescue
+      puts <<-EOS
+ERROR: No rubyforge config file found: #{@config_file}
+Run 'rubyforge setup' to prepare your env for access to Rubyforge
+ - See http://newgem.rubyforge.org/rubyforge.html for more details
+      EOS
+      exit
+    end
+  end
+  RUBYFORGE_USERNAME.replace @config["username"]
+end
+
+
+REV = nil 
+# UNCOMMENT IF REQUIRED: 
+# REV = YAML.load(`svn info`)['Revision']
+VERS = Semantic::VERSION::STRING + (REV ? ".#{REV}" : "")
+RDOC_OPTS = ['--quiet', '--title', 'Rsemantic documentation',
+    "--opname", "index.html",
+    "--line-numbers", 
+    "--main", "README.textile",
+    "--inline-source"]
+
+class Hoe
+  def extra_deps 
+    @extra_deps.reject! { |x| Array(x).first == 'hoe' } 
+    @extra_deps
+  end 
+end
+
+# Generate all the Rake tasks
+# Run 'rake -T' to see list of generated tasks (from gem root directory)
+$hoe = Hoe.new(GEM_NAME, VERS) do |p|
+  p.developer(AUTHOR, EMAIL)
+  p.description = DESCRIPTION
+  p.summary = DESCRIPTION
+  p.url = HOMEPATH
+  p.rubyforge_name = RUBYFORGE_PROJECT if RUBYFORGE_PROJECT
+  p.clean_globs |= ['**/.*.sw?', '*.gem', '.config', '**/.DS_Store', '**/*.class', '**/*.jar']  #An array of file patterns to delete on clean.
+  
+  # == Optional
+  p.changes = p.paragraphs_of("History.txt", 0..1).join("\n\n")
+  #p.extra_deps = []     # An array of rubygem dependencies [name, version], e.g. [ ['active_support', '>= 1.3.1'] ]
+  p.extra_deps = [ ['term-ansicolor', '>= 1.0.3'], ['rspec', '>= 1.1.5'], ['diff-lcs', '>= 1.1.2'] ]
+
+  #p.spec_extras = {}    # A hash of extra values to set in the gemspec.
+  
+end
+
+CHANGES = $hoe.paragraphs_of('History.txt', 0..1).join("\\n\\n")
+PATH    = (RUBYFORGE_PROJECT == GEM_NAME) ? RUBYFORGE_PROJECT : "#{RUBYFORGE_PROJECT}/#{GEM_NAME}"
+$hoe.remote_rdoc_dir = File.join(PATH.gsub(/^#{RUBYFORGE_PROJECT}\/?/,''), 'rdoc')
+$hoe.rsync_args = '-av --delete --ignore-errors'

data/config/requirements.rb

@@ -0,0 +1,15 @@
+require 'fileutils'
+include FileUtils
+
+require 'rubygems'
+%w[rake hoe].each do |req_gem|
+  begin
+    require req_gem
+  rescue LoadError
+    puts "This Rakefile requires the '#{req_gem}' RubyGem."
+    puts "Installation: gem install #{req_gem} -y"
+    exit
+  end
+end
+
+$:.unshift(File.join(File.dirname(__FILE__), %w[.. lib]))

data/gem_tasks/deployment.rake

@@ -0,0 +1,34 @@
+desc 'Release the website and new gem version'
+task :deploy => [:check_version, :website, :release] do
+  puts "Remember to create SVN tag:"
+  puts "svn copy svn+ssh://#{rubyforge_username}@rubyforge.org/var/svn/#{PATH}/trunk " +
+    "svn+ssh://#{rubyforge_username}@rubyforge.org/var/svn/#{PATH}/tags/REL-#{VERS} "
+  puts "Suggested comment:"
+  puts "Tagging release #{CHANGES}"
+end
+
+desc 'Runs tasks website_generate and install_gem as a local deployment of the gem'
+task :local_deploy => [:website_generate, :install_gem]
+
+task :check_version do
+  unless ENV['VERSION']
+    puts 'Must pass a VERSION=x.y.z release version'
+    exit
+  end
+  unless ENV['VERSION'] == VERS
+    puts "Please update your version.rb to match the release version, currently #{VERS}"
+    exit
+  end
+end
+
+desc 'Install the package as a gem, without generating documentation(ri/rdoc)'
+task :install_gem_no_doc => [:clean, :package] do
+  sh "#{'sudo ' unless Hoe::WINDOZE }gem install pkg/*.gem --no-rdoc --no-ri"
+end
+
+namespace :manifest do
+  desc 'Recreate Manifest.txt to include ALL files'
+  task :refresh do
+    `rake check_manifest | patch -p0 > Manifest.txt`
+  end
+end

data/gem_tasks/environment.rake

@@ -0,0 +1,7 @@
+task :ruby_env do
+  RUBY_APP = if RUBY_PLATFORM =~ /java/
+    "jruby"
+  else
+    "ruby"
+  end unless defined? RUBY_APP
+end

data/gem_tasks/examples.rake

@@ -0,0 +1,29 @@
+require 'lib/semantic'
+
+namespace :example do
+
+  documents = ["The cat in the hat disabled", "A cat is a fine pet ponies.", "Dogs and cats make good pets.","I haven't got a hat."]
+
+  desc "run main LSA example"
+  task :lsa do
+    search = Semantic::Search.new(documents, :verbose => true)
+  end
+
+  desc "run main Vector space example"
+  task :vector_space do
+    search = Semantic::Search.new(documents)
+
+    puts "Documents:"
+    documents.each_with_index { |document, index| puts "#{index}: #{document}"  }
+    puts
+
+    puts "Documents related to first document: #{documents[0]}"
+    puts search.related(0)
+    puts
+
+    puts "Searching for the word cat:"
+    puts search.search(["cat"])
+    puts
+  end
+
+end

data/gem_tasks/fix_cr_lf.rake

@@ -0,0 +1,10 @@
+desc 'Make all files use UNIX (\n) line endings'
+task :fix_cr_lf do
+  files = FileList['**/*']
+  files.each do |f|
+    next if File.directory?(f)
+    s = IO.read(f)
+    s.gsub!(/\r?\n/, "\n")
+    File.open(f, "w") { |io| io.write(s) }
+  end
+end

data/gem_tasks/gemspec.rake

@@ -0,0 +1,6 @@
+namespace :gemspec do
+  desc 'Refresh rsemantic.gemspec to include ALL files'
+  task :refresh => 'manifest:refresh' do
+    File.open('rsemantic.gemspec', 'w') {|io| io.write($hoe.spec.to_ruby)}
+  end
+end

data/gem_tasks/rspec.rake

@@ -0,0 +1,33 @@
+begin
+  require 'spec'
+rescue LoadError
+  require 'rubygems'
+  require 'spec'
+end
+begin
+  require 'spec/rake/spectask'
+  require 'spec/rake/verify_rcov'
+rescue LoadError
+  puts <<-EOS
+To use rspec for testing you must install rspec gem:
+    gem install rspec
+EOS
+  exit(0)
+end
+
+desc "Run the specs under spec/models"
+Spec::Rake::SpecTask.new do |t|
+  t.spec_opts = ['--options', "spec/spec.opts"]
+  t.spec_files = FileList['spec/**/*_spec.rb']
+  
+    unless ENV['NO_RCOV']
+      t.rcov = true
+      t.rcov_dir = 'coverage'
+      t.rcov_opts = ['--exclude', '_helper\.rb,_spec\.rb,spec\/boss,\/var\/lib\/gems,\/Library\/Ruby,\.autotest']
+    end
+end
+
+RCov::VerifyTask.new(:verify_rcov => :spec) do |t|
+  t.threshold = 99.7 # Make sure you have rcov 0.9 or higher!
+  t.index_html = 'coverage/index.html'
+end

data/gem_tasks/website.rake

@@ -0,0 +1,17 @@
+desc 'Generate website files'
+task :website_generate => :ruby_env do
+  (Dir['website/**/*.txt'] - Dir['website/version*.txt']).each do |txt|
+    sh %{ #{RUBY_APP} script/txt2html #{txt} > #{txt.gsub(/txt$/,'html')} }
+  end
+end
+
+desc 'Upload website files to rubyforge'
+task :website_upload do
+  host = "#{rubyforge_username}@rubyforge.org"
+  remote_dir = "/var/www/gforge-projects/#{PATH}/"
+  local_dir = 'website'
+  sh %{rsync -aCv #{local_dir}/ #{host}:#{remote_dir}}
+end
+
+desc 'Generate and upload website files'
+task :website => [:website_generate, :website_upload, :publish_docs]

data/lib/semantic/compare.rb

@@ -0,0 +1,19 @@
+module Semantic
+  class Compare
+
+    class << self
+      
+      def similarity(vector1, vector2)
+        cosine(vector1, vector2)
+      end
+      
+      def cosine(vector1, vector2)
+        unless vector2.nil? or vector1.nil?
+          (vector2.dot(vector1)) / (vector1.norm * vector2.norm)
+        end
+      end
+
+    end
+    
+  end
+end

data/lib/semantic/matrix_transformer.rb

@@ -0,0 +1,25 @@
+module Semantic
+  class MatrixTransformer
+
+    def initialize(options={})
+      @transforms = options[:transforms] || [:TFIDF, :LSA]
+      @options = options
+    end
+
+    def apply_transforms(vector_space_model)
+      @transforms.each do |transform|
+        begin
+          transform_class = Semantic::Transform.const_get(transform)
+          Semantic.logger.info("Applying #{transform} transform")
+          vector_space_model.matrix = transform_class.send(:transform, vector_space_model.matrix) if transform_class.respond_to?(:transform)
+          Semantic.logger.info(vector_space_model)
+        rescue Exception => e
+          Semantic.logger.error("Error: Cannot perform transform: #{transform}")
+          Semantic.logger.error(e)
+        end
+      end
+      vector_space_model
+    end
+
+  end
+end

data/lib/semantic/parser.rb

@@ -0,0 +1,40 @@
+require 'stemmer'
+
+module Semantic
+  class Parser
+
+    def initialize
+      #English stopwords from ftp://ftp.cs.cornell.edu/pub/smart/english.stop
+      #TODO: nicer way to reference stop file location?
+      File.open(File.dirname(__FILE__)+'/../../resources/english.stop', 'r') do |file|
+        @stopwords = file.read().split()
+      end
+    end
+
+    def tokenise_and_filter(string)
+      word_list = tokenise_and_stem(string)
+      remove_stop_words(word_list)
+    end
+
+    #remove any nasty grammar tokens from string
+    def clean(string)
+      string = string.gsub(".","")
+      string = string.gsub(/\s+/," ")
+      string = string.downcase
+      return string
+    end
+
+    #stop words are common words which have no search value
+    def remove_stop_words(list)
+      list.select {|word| word unless @stopwords.include? word }
+    end
+
+    def tokenise_and_stem(string)
+      string = clean(string)
+      words = string.split(" ")
+
+      words.map {|word| word.stem }
+    end
+
+  end
+end

data/lib/semantic/search.rb

@@ -0,0 +1,35 @@
+module Semantic
+  class Search
+
+    def initialize(documents, options={})
+      Semantic.logger.level = Logger::INFO if options[:verbose]
+      
+      @builder = VectorSpace::Builder.new(options)
+      @matrix_transformer = MatrixTransformer.new(options)
+
+      @vector_space_model = @builder.build_document_matrix(documents)
+      
+      Semantic.logger.info(@vector_space_model)
+      
+      @vector_space_model = @matrix_transformer.apply_transforms(@vector_space_model)
+    end
+  
+    def related(documentId)
+      ratings = []
+      for index in (0...@vector_space_model.ncol)
+        ratings << Compare.similarity(@vector_space_model.column(documentId), @vector_space_model.column(index))
+      end
+      ratings
+    end
+
+    def search(searchList)
+      ratings = []
+      query_vector = @builder.build_query_vector(searchList)
+      for index in (0...@vector_space_model.ncol)
+        ratings << Compare.similarity(query_vector, @vector_space_model.column(index))
+      end
+      ratings
+    end
+  
+  end
+end

data/lib/semantic/transform/lsa_transform.rb

@@ -0,0 +1,34 @@
+module Semantic
+  module Transform
+    class LSA
+
+      class << self
+
+        # Calculate SVD of objects matrix: U . SIGMA . VT = MATRIX
+        # Reduce the dimension of sigma by specified factor producing sigma'.
+        # Then dot product the matrices:  U . SIGMA' . VT = MATRIX'
+        def transform(matrix, dimensions=1)
+          columns = matrix.num_columns
+
+          if dimensions <= columns: #Its a valid reduction
+
+            u, sigma, vt = matrix.singular_value_decomposition
+
+            #Dimension reduction, build SIGMA'
+            for index in ((columns-dimensions)...columns)
+              sigma[index,index]=0
+            end
+
+            #Reconstruct MATRIX' and Save transform
+            matrix = u * sigma * vt
+
+          else
+            raise Exception, "dimension reduction cannot be greater than %s" % rows
+          end
+          
+          matrix
+        end
+      end
+    end
+  end
+end

data/lib/semantic/transform/tf_idf_transform.rb

@@ -0,0 +1,42 @@
+module Semantic
+  module Transform
+    class TFIDF
+
+      @@number_of_documents_with_term = []
+
+      def self.transform(matrix)
+        number_of_documents = matrix.num_columns
+        @@number_of_documents_with_term = []
+
+        matrix.columns.each_with_index do |document, column_index|
+          document_term_total = document.rows.inject(0.0) {|word_sum, word_count| word_sum + word_count.to_f }
+
+          document.rows.each_with_index do |term_weight, row_index|
+            unless term_weight.to_f == 0.0
+              matrix[row_index, column_index] = (term_weight / document_term_total) *
+              Math.log((number_of_documents / number_of_documents_with_term(row_index, matrix).to_f).abs)
+            end
+          end
+        end
+        matrix
+      end
+
+      def self.number_of_documents_with_term(row_index, matrix)
+        return @@number_of_documents_with_term[row_index] unless @@number_of_documents_with_term[row_index].nil?
+
+        term_document_occurences = 0
+
+        rows,cols = matrix.dimensions
+
+        for n in (0...cols)
+          if matrix[row_index, n] > 0 #Term appears in document
+            term_document_occurences += 1
+          end
+        end
+        @@number_of_documents_with_term[row_index] = term_document_occurences
+        @@number_of_documents_with_term[row_index]
+      end
+
+    end
+  end
+end

data/lib/semantic/transform.rb

@@ -0,0 +1 @@
+%w{tf_idf lsa}.each{|f| require "semantic/transform/#{f}_transform.rb"}

data/lib/semantic/vector_space/builder.rb

@@ -0,0 +1,69 @@
+module Semantic
+  module VectorSpace
+    #A algebraic model for representing text documents as vectors of identifiers.
+    #A document is represented as a vector. Each dimension of the vector corresponds to a
+    #separate term. If a term occurs in the document, then the value in the vector is non-zero.
+    class Builder
+
+      def initialize(options={})
+        @parser = Parser.new
+        @options = options
+        @parsed_document_cache = []
+      end
+
+      def build_document_matrix(documents)
+        @vector_keyword_index = build_vector_keyword_index(documents)
+       
+        document_vectors = documents.enum_for(:each_with_index).map{|document,document_id| build_vector(document, document_id)}
+        document_matrix = Linalg::DMatrix.join_columns(document_vectors)
+        
+        Model.new(document_matrix, @vector_keyword_index)
+      end
+
+      def build_query_vector(term_list)
+        build_vector(term_list.join(" "))
+      end
+
+      private
+      def build_vector_keyword_index(documents)
+        parse_and_cache(documents)
+        vocabulary_list = find_unique_vocabulary
+	      map_vocabulary_to_vector_positions(vocabulary_list)
+      end
+
+      def parse_and_cache(documents)
+        documents.each_with_index do |document, index|
+          @parsed_document_cache[index] = @parser.tokenise_and_filter(document)
+        end
+      end
+
+      def find_unique_vocabulary
+        vocabulary_list = @parsed_document_cache.inject([]) { |parsed_document, vocabulary_list| vocabulary_list + parsed_document  }
+        vocabulary_list.uniq
+      end
+      
+      def map_vocabulary_to_vector_positions(vocabulary_list)
+        vector_index={}
+        column = 0
+	      vocabulary_list.each do |word|
+          vector_index[word] = column
+          column += 1
+        end
+        vector_index
+      end 
+    
+      def build_vector(word_string, document_id=nil)
+        if document_id.nil?
+          word_list = @parser.tokenise_and_filter(word_string)
+        else
+          word_list = @parsed_document_cache[document_id]
+        end
+        
+        vector = Linalg::DMatrix.new(@vector_keyword_index.length, 1)
+        word_list.each { |word| vector[@vector_keyword_index[word] , 0] += 1 if @vector_keyword_index.has_key?(word)  }
+        vector
+      end
+
+    end
+  end
+end

data/lib/semantic/vector_space/model.rb

@@ -0,0 +1,45 @@
+require 'linalg'
+require 'delegate'
+
+module Semantic
+  module VectorSpace
+
+    class Model < DelegateClass(::Linalg::DMatrix)
+
+      def initialize(matrix, keywords)
+        @keywords = keywords || {}
+        super(matrix)
+      end
+      
+      def matrix=(matrix)
+        @_dc_obj = matrix
+      end
+      
+      def matrix
+        @_dc_obj
+      end
+            
+      def to_s
+        out = StringIO.new
+        out.print " " * 9
+
+        matrix.ncol.times do |id|
+          out.print "  D#{id+1}  " 
+        end
+        out.puts
+
+        matrix.rows.each_with_index do |terms, index|
+          out.print "#{@keywords.index(index).ljust(6)}" if @keywords.has_value?(index)
+          out.print "[ "
+          terms.columns.each do |document|
+            out.print "%+0.2f " % document
+          end
+          out.print "]"
+          out.puts
+        end
+        out.string
+      end
+      
+    end
+  end
+end

data/lib/semantic/vector_space.rb

@@ -0,0 +1 @@
+%w{model builder}.each{|f| require "semantic/vector_space/#{f}"}

data/lib/semantic/version.rb

@@ -0,0 +1,9 @@
+module Semantic #:nodoc:
+  class VERSION #:nodoc:
+    MAJOR = 0
+    MINOR = 1
+    TINY  = 0
+
+    STRING = [MAJOR, MINOR, TINY].join('.')
+  end
+end