rsemantic 0.1.3

data/rsemantic.gemspec

@@ -0,0 +1,41 @@
+Gem::Specification.new do |s|
+  s.name = %q{rsemantic}
+  s.version = "0.1.3"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Joseph Wilk"]
+  s.date = %q{2009-08-01}
+  s.description = %q{A document vector search with flexible matrix transforms. Currently supports Latent semantic analysis and Term frequency - inverse document frequency}
+  s.email = ["joe@josephwilk.net"]
+  s.extra_rdoc_files = ["History.txt", "Manifest.txt", "README.txt", "TODO.txt"]
+  s.files = ["History.txt", "Manifest.txt", "README.txt", "Rakefile", "TODO.txt", "config/hoe.rb", "config/requirements.rb", "gem_tasks/deployment.rake", "gem_tasks/environment.rake", "gem_tasks/examples.rake", "gem_tasks/fix_cr_lf.rake", "gem_tasks/gemspec.rake", "gem_tasks/rspec.rake", "gem_tasks/website.rake", "lib/semantic.rb", "lib/semantic/compare.rb", "lib/semantic/matrix_transformer.rb", "lib/semantic/parser.rb", "lib/semantic/search.rb", "lib/semantic/transform.rb", "lib/semantic/transform/lsa_transform.rb", "lib/semantic/transform/tf_idf_transform.rb", "lib/semantic/vector_space.rb", "lib/semantic/vector_space/builder.rb", "lib/semantic/vector_space/model.rb", "lib/semantic/version.rb", "resources/english.stop", "rsemantic.gemspec", "spec/semantic/compare_spec.rb", "spec/semantic/matrix_transformer_spec.rb", "spec/semantic/parser_spec.rb", "spec/semantic/search_spec.rb", "spec/semantic/transform/lsa_transform_spec.rb", "spec/semantic/transform/tf_idf_transform_spec.rb", "spec/semantic/vector_space/builder_spec.rb", "spec/semantic/vector_space/model_spec.rb", "spec/spec.opts", "spec/spec_helper.rb"]
+  s.has_rdoc = true
+  s.homepage = %q{http://github.com/josephwilk/rsemantic}
+  s.rdoc_options = ["--main", "README.txt"]
+  s.require_paths = ["lib"]
+  s.rubyforge_project = %q{rsemantic}
+  s.rubygems_version = %q{1.3.1}
+  s.summary = %q{A document vector search with flexible matrix transforms. Currently supports Latent semantic analysis and Term frequency - inverse document frequency}
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 2
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<term-ansicolor>, [">= 1.0.3"])
+      s.add_runtime_dependency(%q<rspec>, [">= 1.1.5"])
+      s.add_runtime_dependency(%q<diff-lcs>, [">= 1.1.2"])
+      s.add_development_dependency(%q<hoe>, [">= 2.3.2"])
+    else
+      s.add_dependency(%q<term-ansicolor>, [">= 1.0.3"])
+      s.add_dependency(%q<rspec>, [">= 1.1.5"])
+      s.add_dependency(%q<diff-lcs>, [">= 1.1.2"])
+      s.add_dependency(%q<hoe>, [">= 2.3.2"])
+    end
+  else
+    s.add_dependency(%q<term-ansicolor>, [">= 1.0.3"])
+    s.add_dependency(%q<rspec>, [">= 1.1.5"])
+    s.add_dependency(%q<diff-lcs>, [">= 1.1.2"])
+    s.add_dependency(%q<hoe>, [">= 2.3.2"])
+  end
+end

data/spec/semantic/compare_spec.rb

@@ -0,0 +1,16 @@
+require File.dirname(__FILE__) + '/../spec_helper'
+
+module Semantic
+  describe Compare do
+
+    def vector(values)
+      Linalg::DMatrix.columns([values])
+    end
+
+    it "should calculate cosine" do
+      cosine =  Compare.cosine( vector([0.1,0.5]), vector([0.9, 0.3]) )
+      cosine.should be_close(0.4961, 0.0001)
+    end
+
+  end
+end

data/spec/semantic/matrix_transformer_spec.rb

@@ -0,0 +1,51 @@
+require File.dirname(__FILE__) + '/../spec_helper'
+
+module Semantic
+  describe MatrixTransformer do
+
+    def mock_transform
+      @transform ||= mock(Transform)
+    end
+
+    def mock_vector_space
+      mock("vector space", :matrix => Linalg::DMatrix.rows([[1,0],[0,1]]), :matrix= => nil )
+    end
+
+
+    describe "transforming matrix" do
+
+      it "should ignore invalid transform class" do
+        matrix_transformer = MatrixTransformer.new(:transforms => [:FAKE])
+        lambda {
+          matrix_transformer.apply_transforms(mock_vector_space)
+        }.should_not raise_error
+      end
+
+      it "should use defaults transforms in none are specified" do
+        matrix_transformer = MatrixTransformer.new
+        Transform.should_receive(:const_get).with(:LSA).and_return(mock_transform)
+        Transform.should_receive(:const_get).with(:TFIDF).and_return(mock_transform)
+
+        matrix_transformer.apply_transforms(mock_vector_space)
+      end
+
+      it "should send transform message to class to transform matrix" do
+        matrix_transformer = MatrixTransformer.new(:transforms => [:LSA])
+        Transform.stub!(:const_get).and_return(mock_transform)
+
+        mock_transform.should_receive(:transform)
+
+        matrix_transformer.apply_transforms(mock_vector_space)
+      end
+
+      it "should check that transform class is capable of transforming" do
+        matrix_transformer = MatrixTransformer.new(:transforms => [:LSA])
+        Transform.stub!(:const_get).and_return(mock_transform)
+        mock_transform.should_receive(:respond_to?).with(:transform)
+
+        matrix_transformer.apply_transforms(mock_vector_space)
+      end
+
+    end
+  end
+end

data/spec/semantic/parser_spec.rb

@@ -0,0 +1,34 @@
+require File.dirname(__FILE__) + '/../spec_helper'
+
+module Semantic
+  describe Parser do
+
+    it "should remove stop words" do
+      file = mock("file")
+      file.stub!(:read).and_return("a to be")
+      File.stub!(:open).and_yield(file)
+      parser = Parser.new
+
+      parser.remove_stop_words(['a','house']).should == ['house']
+    end
+
+    it "should remove any non characters" do
+      file = mock("file")
+      file.stub!(:read).and_return("a to be")
+      File.stub!(:open).and_yield(file)
+
+      parser = Parser.new
+      parser.tokenise_and_stem("dragon.").should == ["dragon"]
+    end
+
+    it "should tokenise the string" do
+      parser = Parser.new
+
+      parser.stub!(:remove_stop_words).and_return(['mouse','trap'])
+      parser.should_receive(:tokenise_and_stem).and_return(['mouse','trap'])
+
+      parser.tokenise_and_filter(['the mouse trap'])
+    end
+
+  end
+end

data/spec/semantic/search_spec.rb

@@ -0,0 +1,129 @@
+require File.dirname(__FILE__) + '/../spec_helper'
+
+module Semantic
+  describe Search do
+
+    documents = ["The cat in the hat disabled", "A cat is a fine pet ponies.", "Dogs and cats make good pets.","I haven't got a hat."]
+
+    def mock_builder
+      @builder ||= mock(VectorSpace::Builder)
+    end
+
+    def mock_matrix_transformer
+      @matrix_transformer ||= mock(MatrixTransformer)
+    end
+
+    def query_vector
+      @query_vector ||= Linalg::DMatrix.columns([[1,0]])
+    end
+
+    def vector_space_model(stubs = {})
+      @vector_space_model ||= VectorSpace::Model.new(Linalg::DMatrix.rows([[0,1],[1,0]]), {})
+    end
+
+    def matrix(array)
+      Linalg::DMatrix.rows(array)
+    end
+
+    def vector(vector)
+      matrix([vector])
+    end
+      
+    describe "setting up" do
+
+      it "should build the vector space" do
+        VectorSpace::Builder.stub!(:new).and_return(mock_builder)
+        mock_builder.should_receive(:build_document_matrix).with(['test']).and_return(vector_space_model)
+
+        Search.new(['test'])
+      end
+
+      it "should transform matrices" do
+        MatrixTransformer.stub!(:new).and_return(mock_matrix_transformer)
+        VectorSpace::Builder.stub!(:new).and_return(mock_builder)
+        mock_builder.stub!(:build_document_matrix).and_return(vector_space_model)
+        
+        #FIXME: with will not match vector_space_model, requests class Data. Think this is related to Delegate and Rspec
+        mock_matrix_transformer.should_receive(:apply_transforms).with(anything).and_return(vector_space_model)
+
+        Search.new(['test'])
+      end
+
+    end
+
+    describe "searching" do
+
+      it "should map search term to vector space" do
+        VectorSpace::Builder.stub!(:new).and_return(mock_builder)
+        mock_builder.stub!(:build_document_matrix).and_return(vector_space_model)
+
+        mock_builder.should_receive(:build_query_vector).with("cat").and_return(query_vector)
+
+        vector_search = Search.new(documents)
+        vector_search.search("cat")
+      end
+
+      it "should compare the documents using cosine" do
+        pending
+      end
+
+    end
+
+    describe "relating" do
+
+      it "should find related documents by comparing cosine" do
+        VectorSpace::Builder.stub!(:new).and_return(mock_builder)
+      
+        mock_builder.stub!(:build_document_matrix).and_return(vector_space_model)
+
+        MatrixTransformer.stub!(:new).and_return(mock_matrix_transformer)
+        mock_matrix_transformer.stub!(:apply_transforms).and_return(vector_space_model)
+
+        Compare.should_receive(:cosine).with(matrix([[0],[1]]), matrix([[0],[1]]))
+        Compare.should_receive(:cosine).with(matrix([[0],[1]]), matrix([[1],[0]]))
+
+        vector_search = Search.new(documents)
+
+        vector_search.related(0)
+      end
+
+    end
+
+    describe "logging" do
+
+      before(:each) do
+        @out = StringIO.new
+        Semantic.logger = Logger.new(@out)
+      end
+
+      it "should set info level if in verbose mode" do
+        VectorSpace::Builder.stub!(:new).and_return(mock_builder)
+        mock_builder.stub!(:build_document_matrix).and_return(vector_space_model)
+
+        Search.new(['test'], :verbose => true)
+        
+        Semantic.logger.level.should == Logger::INFO
+      end
+      
+      it "should set error level if not in verbose mode" do
+        VectorSpace::Builder.stub!(:new).and_return(mock_builder)
+        mock_builder.stub!(:build_document_matrix).and_return(vector_space_model)
+
+        Search.new(['test'], :verbose => false)
+        
+        Semantic.logger.level.should == Logger::ERROR
+      end
+
+      it "should default to error level if verbose is not specified" do
+        VectorSpace::Builder.stub!(:new).and_return(mock_builder)
+        mock_builder.stub!(:build_document_matrix).and_return(vector_space_model)
+
+        Search.new(['test'])
+        
+        Semantic.logger.level.should == Logger::ERROR
+      end
+
+    end
+
+  end
+end

data/spec/semantic/transform/lsa_transform_spec.rb

@@ -0,0 +1,59 @@
+require File.dirname(__FILE__) + '/../../spec_helper'
+
+module Semantic
+  describe Transform::LSA do
+
+    tiny_matrix = Linalg::DMatrix.columns([[0.0, 1.0, 0.0],
+                                           [1.0, 0.0, 1.0]])
+
+    u = Linalg::DMatrix.rows([[1,0],
+                              [0,1]])
+
+    vt = Linalg::DMatrix.rows([[1,0,0],
+                               [1,0,0],
+                               [1,0,0]])
+
+    sigma = Linalg::DMatrix.rows([[1,0,0],
+                                  [0,1,0]])
+
+    describe "latent semantic analysis transform" do
+
+      it "should use svd on matrix" do
+        matrix = Linalg::DMatrix.columns([[0.0, 1.0, 0.0],
+                                          [1.0, 0.0, 1.0]])
+
+        matrix.should_receive(:singular_value_decomposition).and_return([u, sigma, vt])
+
+        Linalg::DMatrix.stub!(:columns).and_return(matrix)
+
+        Transform::LSA.transform(matrix)
+      end
+
+      it "should reduce the noise in the sigma matrix" do
+        matrix = Linalg::DMatrix.columns([[0.0, 1.0, 0.0],
+                                          [1.0, 0.0, 1.0]])
+
+        matrix.stub!(:singular_value_decomposition).and_return([u, sigma, vt])
+        Linalg::DMatrix.stub!(:columns).and_return(matrix)
+
+        sigma.should_receive(:[]=).with(0,0,0)
+        sigma.should_receive(:[]=).with(1,1,0)
+
+        Transform::LSA.transform(matrix, 2)
+      end
+
+      it "should prevent reducing dimensions greater than the matrixes own dimensions" do
+        lambda { Transform::LSA.transform tiny_matrix, 100 }.should raise_error(Exception)
+      end
+
+      it "should transform LSA matrix" do
+        transformed_matrix = Transform::LSA.transform tiny_matrix
+
+        #TODO: better way to compare result matrix
+        transformed_matrix.to_s.should == Linalg::DMatrix.columns([[0,0,0],[1,0,1]]).to_s
+      end
+
+    end
+
+  end
+end

data/spec/semantic/transform/tf_idf_transform_spec.rb

@@ -0,0 +1,35 @@
+require File.dirname(__FILE__) + '/../../spec_helper'
+
+module Semantic
+  describe Transform::TFIDF do
+
+    def matrix(matrix)
+      Linalg::DMatrix.rows(matrix)
+    end
+
+    tiny_matrix = Linalg::DMatrix.rows([[0.0, 1.0, 0.0],
+    [1.0, 0.0, 1.0]])
+
+    describe "term frequency / inverse document frequency transform" do
+
+      it "should find the number of times each term occurs" do
+        Transform::TFIDF.should_receive(:number_of_documents_with_term).with(0, matrix([[1]])).and_return(2)
+
+        Transform::TFIDF.transform(matrix([[1]]))
+      end
+
+      it "should ignore counting terms with 0 weighting" do
+        Transform::TFIDF.should_not_receive(:number_of_documents_with_term)
+
+        Transform::TFIDF.transform(matrix([[0,0],[0,0]]))
+      end
+
+      it "should calculate term frequency * inverse document freuency" do
+        transformed_matrix = Transform::TFIDF.transform matrix([[1,1],[0,1]])
+
+        transformed_matrix.to_s.should == Linalg::DMatrix.columns([[0, 0],[0, 0.346574]]).to_s
+      end
+
+    end
+  end
+end

data/spec/semantic/vector_space/builder_spec.rb

@@ -0,0 +1,44 @@
+require File.dirname(__FILE__) + '/../../spec_helper'
+
+module Semantic
+  module VectorSpace
+    describe Builder do
+
+      def mock_parser
+        @parser ||= mock("Parser")
+      end
+
+      def documents
+        ['nipon','ichiban']
+      end
+
+
+      describe "building query vector" do
+
+        it "should build vector from string" do
+          builder = Builder.new
+          builder.should_receive(:build_vector).with("query string")
+
+          builder.build_query_vector(["query","string"])
+        end
+
+        it "should generate a valid vector" do
+          builder = Builder.new
+          builder.build_document_matrix(["query string"])
+          query = builder.build_query_vector(["query","string"])
+
+          query.should == Linalg::DMatrix.columns([[1,1]])
+        end
+
+        it "should generate empty vector when terms are not in document matrix" do
+          builder = Builder.new
+          builder.build_document_matrix(["string"])
+          query = builder.build_query_vector(["not-in-document"])
+
+          query.should == Linalg::DMatrix.columns([[0]])
+        end
+
+      end
+    end
+  end
+end

data/spec/semantic/vector_space/model_spec.rb

@@ -0,0 +1,22 @@
+require File.dirname(__FILE__) + '/../../spec_helper'
+
+module Semantic
+  module VectorSpace
+
+    describe Model do
+
+      it "should output a DMatrix as a pretty string" do
+        model = Model.new(Linalg::DMatrix.columns([[0.11111,0.66666],[0.33333, 0.001]]), {})
+
+        model.to_s.should include("[ +0.11 +0.33 ]\n[ +0.67 +0.00 ]\n")
+      end
+      
+      it "should output keywords for the matrix rows" do
+        model = Model.new(Linalg::DMatrix.columns([[0]]), {'shiva' => 0})
+        
+        model.to_s.should include("shiva [ +0.00 ]")
+      end
+
+    end
+  end
+end

data/spec/spec.opts

@@ -0,0 +1,2 @@
+--colour
+--diff