josephwilk-rsemantic 0.1.0

data/spec/semantic/search_spec.rb

@@ -0,0 +1,93 @@
+require File.dirname(__FILE__) + '/../spec_helper'
+
+module Semantic
+  describe Search do
+
+    documents = ["The cat in the hat disabled", "A cat is a fine pet ponies.", "Dogs and cats make good pets.","I haven't got a hat."]
+
+    def mock_builder
+      @builder ||= mock(VectorSpace::Builder)
+    end
+
+    def mock_matrix_transformer
+      @matrix_transformer ||= mock(MatrixTransformer)
+    end
+
+    def query_vector
+      @query_vector ||= Linalg::DMatrix.columns([[1,0]])
+    end
+
+    def vector_space_model(stubs = {})
+      @vector_space_model ||= VectorSpace::Model.new(Linalg::DMatrix.rows([[0,1],[1,0]]), [])
+    end
+
+    def matrix(array)
+      Linalg::DMatrix.rows(array)
+    end
+
+    def vector(vector)
+      matrix([vector])
+    end
+      
+    describe "setting up" do
+
+      it "should build the vector space" do
+        VectorSpace::Builder.stub!(:new).and_return(mock_builder)
+        mock_builder.should_receive(:build_document_matrix).with(['test']).and_return(vector_space_model)
+
+        Search.new(['test'])
+      end
+
+      it "should transform matrices" do
+        MatrixTransformer.stub!(:new).and_return(mock_matrix_transformer)
+        VectorSpace::Builder.stub!(:new).and_return(mock_builder)
+        mock_builder.stub!(:build_document_matrix).and_return(vector_space_model)
+        
+        #FIXME: with will not match vector_space_model, requests class Data. Think this is related to Delegate and Rspec
+        mock_matrix_transformer.should_receive(:apply_transforms).with(anything).and_return(vector_space_model)
+
+        Search.new(['test'])
+      end
+
+    end
+
+    describe "searching" do
+
+      it "should map search term to vector space" do
+        VectorSpace::Builder.stub!(:new).and_return(mock_builder)
+        mock_builder.stub!(:build_document_matrix).and_return(vector_space_model)
+
+        mock_builder.should_receive(:build_query_vector).with("cat").and_return(query_vector)
+
+        vector_search = Search.new(documents)
+        vector_search.search("cat")
+      end
+
+      it "should compare the documents using cosine" do
+        pending
+      end
+
+    end
+
+    describe "relating" do
+
+      it "should find related documents by comparing cosine" do
+        VectorSpace::Builder.stub!(:new).and_return(mock_builder)
+      
+        mock_builder.stub!(:build_document_matrix).and_return(vector_space_model)
+
+        MatrixTransformer.stub!(:new).and_return(mock_matrix_transformer)
+        mock_matrix_transformer.stub!(:apply_transforms).and_return(vector_space_model)
+
+        Compare.should_receive(:cosine).with(matrix([[0],[1]]), matrix([[0],[1]]))
+        Compare.should_receive(:cosine).with(matrix([[0],[1]]), matrix([[1],[0]]))
+
+        vector_search = Search.new(documents)
+
+        vector_search.related(0)
+      end
+
+    end
+
+  end
+end

data/spec/semantic/transform/lsa_transform_spec.rb

@@ -0,0 +1,59 @@
+require File.dirname(__FILE__) + '/../../spec_helper'
+
+module Semantic
+  describe Transform::LSA do
+
+    tiny_matrix = Linalg::DMatrix.columns([[0.0, 1.0, 0.0],
+                                           [1.0, 0.0, 1.0]])
+
+    u = Linalg::DMatrix.rows([[1,0],
+                              [0,1]])
+
+    vt = Linalg::DMatrix.rows([[1,0,0],
+                               [1,0,0],
+                               [1,0,0]])
+
+    sigma = Linalg::DMatrix.rows([[1,0,0],
+                                  [0,1,0]])
+
+    describe "latent semantic analysis transform" do
+
+      it "should use svd on matrix" do
+        matrix = Linalg::DMatrix.columns([[0.0, 1.0, 0.0],
+                                          [1.0, 0.0, 1.0]])
+
+        matrix.should_receive(:singular_value_decomposition).and_return([u, sigma, vt])
+
+        Linalg::DMatrix.stub!(:columns).and_return(matrix)
+
+        Transform::LSA.transform(matrix)
+      end
+
+      it "should reduce the noise in the sigma matrix" do
+        matrix = Linalg::DMatrix.columns([[0.0, 1.0, 0.0],
+                                          [1.0, 0.0, 1.0]])
+
+        matrix.stub!(:singular_value_decomposition).and_return([u, sigma, vt])
+        Linalg::DMatrix.stub!(:columns).and_return(matrix)
+
+        sigma.should_receive(:[]=).with(0,0,0)
+        sigma.should_receive(:[]=).with(1,1,0)
+
+        Transform::LSA.transform(matrix, 2)
+      end
+
+      it "should prevent reducing dimensions greater than the matrixes own dimensions" do
+        lambda { Transform::LSA.transform tiny_matrix, 100 }.should raise_error(Exception)
+      end
+
+      it "should transform LSA matrix" do
+        transformed_matrix = Transform::LSA.transform tiny_matrix
+
+        #TODO: better way to compare result matrix
+        transformed_matrix.to_s.should == Linalg::DMatrix.columns([[0,0,0],[1,0,1]]).to_s
+      end
+
+    end
+
+  end
+end

data/spec/semantic/transform/tf_idf_transform_spec.rb

@@ -0,0 +1,35 @@
+require File.dirname(__FILE__) + '/../../spec_helper'
+
+module Semantic
+  describe Transform::TFIDF do
+
+    def matrix(matrix)
+      Linalg::DMatrix.rows(matrix)
+    end
+
+    tiny_matrix = Linalg::DMatrix.rows([[0.0, 1.0, 0.0],
+    [1.0, 0.0, 1.0]])
+
+    describe "term frequency / inverse document frequency transform" do
+
+      it "should find the number of times each term occurs" do
+        Transform::TFIDF.should_receive(:number_of_documents_with_term).with(0, matrix([[1]])).and_return(2)
+
+        Transform::TFIDF.transform(matrix([[1]]))
+      end
+
+      it "should ignore counting terms with 0 weighting" do
+        Transform::TFIDF.should_not_receive(:number_of_documents_with_term)
+
+        Transform::TFIDF.transform(matrix([[0,0],[0,0]]))
+      end
+
+      it "should calculate term frequency * inverse document freuency" do
+        transformed_matrix = Transform::TFIDF.transform matrix([[1,1],[0,1]])
+
+        transformed_matrix.to_s.should == Linalg::DMatrix.columns([[0, 0],[0, 0.346574]]).to_s
+      end
+
+    end
+  end
+end

data/spec/semantic/vector_space/builder_spec.rb

@@ -0,0 +1,44 @@
+require File.dirname(__FILE__) + '/../../spec_helper'
+
+module Semantic
+  module VectorSpace
+    describe Builder do
+
+      def mock_parser
+        @parser ||= mock("Parser")
+      end
+
+      def documents
+        ['nipon','ichiban']
+      end
+
+
+      describe "building query vector" do
+
+        it "should build vector from string" do
+          builder = Builder.new
+          builder.should_receive(:build_vector).with("query string")
+
+          builder.build_query_vector(["query","string"])
+        end
+
+        it "should generate a valid vector" do
+          builder = Builder.new
+          builder.build_document_matrix(["query string"])
+          query = builder.build_query_vector(["query","string"])
+
+          query.should == Linalg::DMatrix.columns([[1,1]])
+        end
+
+        it "should generate empty vector when terms are not in document matrix" do
+          builder = Builder.new
+          builder.build_document_matrix(["string"])
+          query = builder.build_query_vector(["not-in-document"])
+
+          query.should == Linalg::DMatrix.columns([[0]])
+        end
+
+      end
+    end
+  end
+end

data/spec/semantic/vector_space/model_spec.rb

@@ -0,0 +1,22 @@
+require File.dirname(__FILE__) + '/../../spec_helper'
+
+module Semantic
+  module VectorSpace
+
+    describe Model do
+
+      it "should output a DMatrix as a pretty string" do
+        model = Model.new(Linalg::DMatrix.columns([[0.11111,0.66666],[0.33333, 0.001]]), {})
+
+        model.to_s.should include("[ +0.11 +0.33 ]\n[ +0.67 +0.00 ]\n")
+      end
+      
+      it "should output keywords for the matrix rows" do
+        model = Model.new(Linalg::DMatrix.columns([[0]]), {'shiva' => 0})
+        
+        model.to_s.should include("shiva [ +0.00 ]")
+      end
+
+    end
+  end
+end

data/spec/spec.opts

@@ -0,0 +1,2 @@
+--colour
+--diff

data/spec/spec_helper.rb

@@ -0,0 +1,7 @@
+require 'rubygems'
+gem 'rspec'
+require 'spec'
+
+$:.unshift(File.join(File.dirname(__FILE__), %w[.. lib]))
+
+require 'semantic'

metadata

@@ -0,0 +1,130 @@
+--- !ruby/object:Gem::Specification 
+name: josephwilk-rsemantic
+version: !ruby/object:Gem::Version 
+  version: 0.1.0
+platform: ruby
+authors: 
+- Joseph Wilk
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2008-11-13 00:00:00 -08:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: term-ansicolor
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.0.3
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.1.5
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: diff-lcs
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.1.2
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: hoe
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.8.2
+    version: 
+description: A document vector search with flexible matrix transforms. Currently supports Latent semantic analysis and Term frequency - inverse document frequency
+email: 
+- josephwilk@joesniff.co.uk
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- History.txt
+- Manifest.txt
+- README.txt
+- TODO.txt
+files: 
+- History.txt
+- Manifest.txt
+- README.txt
+- Rakefile
+- TODO.txt
+- config/hoe.rb
+- config/requirements.rb
+- gem_tasks/deployment.rake
+- gem_tasks/environment.rake
+- gem_tasks/examples.rake
+- gem_tasks/fix_cr_lf.rake
+- gem_tasks/gemspec.rake
+- gem_tasks/rspec.rake
+- gem_tasks/website.rake
+- lib/semantic.rb
+- lib/semantic/compare.rb
+- lib/semantic/matrix_transformer.rb
+- lib/semantic/parser.rb
+- lib/semantic/search.rb
+- lib/semantic/transform.rb
+- lib/semantic/transform/lsa_transform.rb
+- lib/semantic/transform/tf_idf_transform.rb
+- lib/semantic/vector_space.rb
+- lib/semantic/vector_space/builder.rb
+- lib/semantic/vector_space/model.rb
+- lib/semantic/version.rb
+- resources/english.stop
+- rsemantic.gemspec
+- spec/semantic/compare_spec.rb
+- spec/semantic/matrix_transformer_spec.rb
+- spec/semantic/parser_spec.rb
+- spec/semantic/search_spec.rb
+- spec/semantic/transform/lsa_transform_spec.rb
+- spec/semantic/transform/tf_idf_transform_spec.rb
+- spec/semantic/vector_space/builder_spec.rb
+- spec/semantic/vector_space/model_spec.rb
+- spec/spec.opts
+- spec/spec_helper.rb
+has_rdoc: true
+homepage: http://github.com/josephwilk/rsemantic
+post_install_message: 
+rdoc_options: 
+- --main
+- README.txt
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: rsemantic
+rubygems_version: 1.2.0
+signing_key: 
+specification_version: 2
+summary: A document vector search with flexible matrix transforms. Currently supports Latent semantic analysis and Term frequency - inverse document frequency
+test_files: []
+