rsemantic 0.1.3 → 0.1.4

data/lib/semantic/transform/tf_idf_transform.rb

@@ -3,38 +3,27 @@ module Semantic
     class TFIDF
 
       @@number_of_documents_with_term = []
-
-      def self.transform(matrix)
-        number_of_documents = matrix.num_columns
+      def self.transform!(matrix)
+        number_of_documents = matrix.size2
         @@number_of_documents_with_term = []
 
-        matrix.columns.each_with_index do |document, column_index|
-          document_term_total = document.rows.inject(0.0) {|word_sum, word_count| word_sum + word_count.to_f }
+        matrix.transpose.enum_for(:each_row).with_index do |document, column_index|
+          document_term_total = document.sum
+
+          document.enum_for(:each).with_index do |term_weight, row_index|
+            unless term_weight == 0.0
+              inverse_document_frequency = GSL::Sf.log((number_of_documents /
+                                                        number_of_documents_with_term(row_index, matrix).to_f).abs)
+              term_frequency = (term_weight / document_term_total)
 
-          document.rows.each_with_index do |term_weight, row_index|
-            unless term_weight.to_f == 0.0
-              matrix[row_index, column_index] = (term_weight / document_term_total) *
-              Math.log((number_of_documents / number_of_documents_with_term(row_index, matrix).to_f).abs)
+              matrix[row_index, column_index] = term_frequency * inverse_document_frequency
             end
           end
         end
-        matrix
       end
 
       def self.number_of_documents_with_term(row_index, matrix)
-        return @@number_of_documents_with_term[row_index] unless @@number_of_documents_with_term[row_index].nil?
-
-        term_document_occurences = 0
-
-        rows,cols = matrix.dimensions
-
-        for n in (0...cols)
-          if matrix[row_index, n] > 0 #Term appears in document
-            term_document_occurences += 1
-          end
-        end
-        @@number_of_documents_with_term[row_index] = term_document_occurences
-        @@number_of_documents_with_term[row_index]
+        @@number_of_documents_with_term[row_index] ||= matrix.row(row_index).where.size
       end
 
     end

data/lib/semantic/vector_space/builder.rb

@@ -1,22 +1,26 @@
 module Semantic
   module VectorSpace
-    #A algebraic model for representing text documents as vectors of identifiers.
-    #A document is represented as a vector. Each dimension of the vector corresponds to a
-    #separate term. If a term occurs in the document, then the value in the vector is non-zero.
+    # A algebraic model for representing text documents as vectors of identifiers.
+    # A document is represented as a vector. Each dimension of the vector corresponds to a
+    # separate term. If a term occurs in the document, then the value in the vector is non-zero.
     class Builder
 
-      def initialize(options={})
-        @parser = Parser.new
-        @options = options
+      def initialize(options = {})
+        @parser = Parser.new(:filter_stop_words => options[:filter_stop_words])
         @parsed_document_cache = []
       end
 
       def build_document_matrix(documents)
         @vector_keyword_index = build_vector_keyword_index(documents)
-       
+
         document_vectors = documents.enum_for(:each_with_index).map{|document,document_id| build_vector(document, document_id)}
-        document_matrix = Linalg::DMatrix.join_columns(document_vectors)
-        
+
+        n = document_vectors.size
+        m = document_vectors.first.size
+
+        # TODO check where else we use document_vectors and if we can directly use column based ones
+        document_matrix = GSL::Matrix.alloc(*document_vectors.map {|v| v.transpose})
+
         Model.new(document_matrix, @vector_keyword_index)
       end
 
@@ -28,7 +32,7 @@ module Semantic
       def build_vector_keyword_index(documents)
         parse_and_cache(documents)
         vocabulary_list = find_unique_vocabulary
-	      map_vocabulary_to_vector_positions(vocabulary_list)
+        map_vocabulary_to_vector_positions(vocabulary_list)
       end
 
       def parse_and_cache(documents)
@@ -38,32 +42,35 @@ module Semantic
       end
 
       def find_unique_vocabulary
-        vocabulary_list = @parsed_document_cache.inject([]) { |parsed_document, vocabulary_list| vocabulary_list + parsed_document  }
-        vocabulary_list.uniq
+        @parsed_document_cache.flatten.reverse.uniq
       end
-      
+
       def map_vocabulary_to_vector_positions(vocabulary_list)
         vector_index={}
         column = 0
-	      vocabulary_list.each do |word|
+        vocabulary_list.each do |word|
           vector_index[word] = column
           column += 1
         end
         vector_index
-      end 
-    
-      def build_vector(word_string, document_id=nil)
+      end
+
+      def build_vector(word_string, document_id = nil)
         if document_id.nil?
           word_list = @parser.tokenise_and_filter(word_string)
         else
           word_list = @parsed_document_cache[document_id]
         end
-        
-        vector = Linalg::DMatrix.new(@vector_keyword_index.length, 1)
-        word_list.each { |word| vector[@vector_keyword_index[word] , 0] += 1 if @vector_keyword_index.has_key?(word)  }
+
+        vector = GSL::Vector.alloc(@vector_keyword_index.length)
+        word_list.each { |word|
+          if @vector_keyword_index.has_key?(word)
+            vector[@vector_keyword_index[word]] += 1
+          end
+        }
+
         vector
       end
-
     end
   end
-end
+end

data/lib/semantic/vector_space/model.rb

@@ -1,39 +1,40 @@
-require 'linalg'
+require 'gsl'
 require 'delegate'
 require 'stringio'
 
 module Semantic
   module VectorSpace
 
-    class Model < DelegateClass(::Linalg::DMatrix)
-
+    class Model < DelegateClass(::GSL::Matrix)
       def initialize(matrix, keywords)
         @keywords = keywords || {}
         @_dc_obj = matrix
         super(matrix)
       end
-      
+
       def matrix=(matrix)
         @_dc_obj = matrix
       end
-      
+
       def matrix
         @_dc_obj
       end
-            
+
       def to_s
         out = StringIO.new
         out.print " " * 9
-        
-        matrix.ncol.times do |id|
-          out.print "  D#{id+1}  " 
+
+        matrix.size2.times do |id|
+          out.print "  D#{id+1}  "
         end
         out.puts
 
-        matrix.rows.each_with_index do |terms, index|
-          out.print "#{@keywords.index(index).ljust(6)}" if @keywords.has_value?(index)
+        matrix.to_a.each_with_index do |terms, index|
+          # TODO fix for 1.8.7
+          out.print "#{@keywords.key(index).ljust(6)}" if @keywords.has_value?(index)
           out.print "[ "
-          terms.columns.each do |document|
+
+          terms.each do |document|
             out.print "%+0.2f " % document
           end
           out.print "]"
@@ -41,7 +42,7 @@ module Semantic
         end
         out.string
       end
-      
+
     end
   end
 end

data/lib/semantic/version.rb

@@ -2,7 +2,7 @@ module Semantic #:nodoc:
   class VERSION #:nodoc:
     MAJOR = 0
     MINOR = 1
-    TINY  = 3
+    TINY  = 4
 
     STRING = [MAJOR, MINOR, TINY].join('.')
   end

data/lib/tasks/rspec.rake

@@ -0,0 +1,13 @@
+require 'rspec/core/rake_task'
+
+desc 'Default: run specs.'
+task :default => :spec
+
+desc "Run specs"
+RSpec::Core::RakeTask.new 
+
+desc "Generate code coverage"
+RSpec::Core::RakeTask.new(:coverage) do |t|
+  t.rcov = true
+  t.rcov_opts = ['--exclude', 'spec']
+end

metadata

@@ -1,136 +1,104 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: rsemantic
-version: !ruby/object:Gem::Version 
-  version: 0.1.3
+version: !ruby/object:Gem::Version
+  version: 0.1.4
+  prerelease: 
 platform: ruby
-authors: 
+authors:
 - Joseph Wilk
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2009-08-01 00:00:00 +01:00
-default_executable: 
-dependencies: 
-- !ruby/object:Gem::Dependency 
-  name: term-ansicolor
+date: 2012-09-18 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: gsl
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.14.7
   type: :runtime
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        version: 1.0.3
-    version: 
-- !ruby/object:Gem::Dependency 
-  name: rspec
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.14.7
+- !ruby/object:Gem::Dependency
+  name: stemmer
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.0.1
   type: :runtime
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        version: 1.1.5
-    version: 
-- !ruby/object:Gem::Dependency 
-  name: diff-lcs
-  type: :runtime
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        version: 1.1.2
-    version: 
-- !ruby/object:Gem::Dependency 
-  name: hoe
-  type: :development
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        version: 2.3.2
-    version: 
-description: A document vector search with flexible matrix transforms. Currently supports Latent semantic analysis and Term frequency - inverse document frequency
-email: 
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.0.1
+description: A document vector search with flexible matrix transforms. Currently supports
+  Latent semantic analysis and Term frequency - inverse document frequency
+email:
 - joe@josephwilk.net
 executables: []
-
 extensions: []
-
-extra_rdoc_files: 
-- History.txt
-- Manifest.txt
-- README.txt
-- TODO.txt
-files: 
+extra_rdoc_files:
 - History.txt
-- Manifest.txt
-- README.txt
-- Rakefile
+- README.md
 - TODO.txt
-- config/hoe.rb
-- config/requirements.rb
-- gem_tasks/deployment.rake
-- gem_tasks/environment.rake
-- gem_tasks/examples.rake
-- gem_tasks/fix_cr_lf.rake
-- gem_tasks/gemspec.rake
-- gem_tasks/rspec.rake
-- gem_tasks/website.rake
-- lib/semantic.rb
+files:
 - lib/semantic/compare.rb
+- lib/semantic/corpus.rb
+- lib/semantic/document.rb
 - lib/semantic/matrix_transformer.rb
 - lib/semantic/parser.rb
 - lib/semantic/search.rb
-- lib/semantic/transform.rb
+- lib/semantic/search_result.rb
 - lib/semantic/transform/lsa_transform.rb
 - lib/semantic/transform/tf_idf_transform.rb
-- lib/semantic/vector_space.rb
+- lib/semantic/transform.rb
 - lib/semantic/vector_space/builder.rb
 - lib/semantic/vector_space/model.rb
+- lib/semantic/vector_space.rb
 - lib/semantic/version.rb
+- lib/semantic.rb
+- lib/tasks/rspec.rake
 - resources/english.stop
-- rsemantic.gemspec
-- spec/semantic/compare_spec.rb
-- spec/semantic/matrix_transformer_spec.rb
-- spec/semantic/parser_spec.rb
-- spec/semantic/search_spec.rb
-- spec/semantic/transform/lsa_transform_spec.rb
-- spec/semantic/transform/tf_idf_transform_spec.rb
-- spec/semantic/vector_space/builder_spec.rb
-- spec/semantic/vector_space/model_spec.rb
-- spec/spec.opts
-- spec/spec_helper.rb
-has_rdoc: true
+- History.txt
+- README.md
+- TODO.txt
 homepage: http://github.com/josephwilk/rsemantic
-licenses: []
-
+licenses:
+- MIT
 post_install_message: 
-rdoc_options: 
-- --main
-- README.txt
-require_paths: 
+rdoc_options:
+- --charset=UTF-8
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      version: "0"
-  version: 
-required_rubygems_version: !ruby/object:Gem::Requirement 
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      version: "0"
-  version: 
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
-rubyforge_project: rsemantic
-rubygems_version: 1.3.5
+rubyforge_project: 
+rubygems_version: 1.8.24
 signing_key: 
-specification_version: 2
-summary: A document vector search with flexible matrix transforms. Currently supports Latent semantic analysis and Term frequency - inverse document frequency
+specification_version: 3
+summary: A document vector search with flexible matrix transforms. Currently supports
+  Latent semantic analysis and Term frequency - inverse document frequency
 test_files: []
-

data/Manifest.txt

@@ -1,38 +0,0 @@
-History.txt
-Manifest.txt
-README.txt
-Rakefile
-TODO.txt
-config/hoe.rb
-config/requirements.rb
-gem_tasks/deployment.rake
-gem_tasks/environment.rake
-gem_tasks/examples.rake
-gem_tasks/fix_cr_lf.rake
-gem_tasks/gemspec.rake
-gem_tasks/rspec.rake
-gem_tasks/website.rake
-lib/semantic.rb
-lib/semantic/compare.rb
-lib/semantic/matrix_transformer.rb
-lib/semantic/parser.rb
-lib/semantic/search.rb
-lib/semantic/transform.rb
-lib/semantic/transform/lsa_transform.rb
-lib/semantic/transform/tf_idf_transform.rb
-lib/semantic/vector_space.rb
-lib/semantic/vector_space/builder.rb
-lib/semantic/vector_space/model.rb
-lib/semantic/version.rb
-resources/english.stop
-rsemantic.gemspec
-spec/semantic/compare_spec.rb
-spec/semantic/matrix_transformer_spec.rb
-spec/semantic/parser_spec.rb
-spec/semantic/search_spec.rb
-spec/semantic/transform/lsa_transform_spec.rb
-spec/semantic/transform/tf_idf_transform_spec.rb
-spec/semantic/vector_space/builder_spec.rb
-spec/semantic/vector_space/model_spec.rb
-spec/spec.opts
-spec/spec_helper.rb