vss 0.1.0

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2010 Mark Dodwell, mkdynamic
+ 
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+ 
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+ 
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Manifest

@@ -0,0 +1,7 @@
+LICENSE
+Manifest
+README.md
+Rakefile
+lib/vss.rb
+lib/vss/engine.rb
+lib/vss/tokenizer.rb

data/README.md

@@ -0,0 +1,41 @@
+# VSS - Vector Space Search 
+
+A simple vector space search engine with **tf*idf** ranking.
+
+## Install
+
+Just install the gem:
+
+    gem install vss
+
+## Usage
+
+To perform a search on a collection of documents:
+
+    require "vss"
+    docs = ["hello", "goodbye", "hello and goodbye", "hello, hello!"]
+    engine = VSS::Engine.new(docs)
+    engine.search("hello") #=> ["hello", "hello, hello!", "hello and goodbye", "goodbye"]
+    
+## Rails/ActiveRecord
+
+If you want to search a collection of `ActiveRecord` objects, you need to pass a **documentizer** `proc` when initializing `VSS::Engine` which will convert the objects into documents (which are simply strings). For example:
+
+    class Page < ActiveRecord::Base
+        #attrs: title, content
+    end
+
+    docs = Page.all
+    documentizer = proc { |record| record.title + " " + record.content }
+    engine = VSS::Engine.new(docs, documentizer)
+        
+## Notes
+
+This isn't designed to be used on huge collections of records. The original use case was for ranking a smallish set of `ActiveRecord` results obtained via a query (using **SearchLogic**). So, essentially, the search consisted of 2 stages; getting the *corpus* via a SQL query, then doing the VSS on that.
+
+## Credits
+
+Heavily inspired by [Joesph Wilk's article on building a vector space search engine in Perl](http://blog.josephwilk.net/projects/building-a-vector-space-search-engine-in-python.html).
+
+Written by Mark Dodwell
+([Design & Code](http://madeofcode.com))

data/Rakefile

@@ -0,0 +1,9 @@
+require "echoe"  
+  
+Echoe.new("vss", "0.1.0") do |p|  
+  p.description = "Simple vector space search engine"    
+  p.url = "http://github.com/mkdynamic/vss"   
+  p.author = "Mark Dodwell"      
+  p.email = "labs@mkdynamic.co.uk"     
+  p.runtime_dependencies = ["stemmer >= 1.0.1"]
+end  

data/lib/vss/engine.rb

@@ -0,0 +1,102 @@
+require "matrix"
+require "vss/tokenizer"
+
+module VSS
+  class Engine    
+    # `documentizer` just takes a record and converts it to a string
+    def initialize(records, documentizer = proc { |document| document })
+      @records = records    
+      @documents = records.map { |record| documentizer.call(record) }
+      @vocab = tokenize(@documents.join(" "))
+    end
+
+    def search(query)
+      # get ranks
+      query_vector = make_query_vector(query)
+      ranks = @documents.map do |document|
+        document_vector = make_vector(document)
+        cosine_rank(query_vector, document_vector)
+      end
+    
+      # now annotate records and return them
+      @records.each_with_index do |record, i|
+        # TODO: do this in a sensible way...
+        record.instance_eval %{def rank; #{ranks[i]}; end}
+      end
+    
+      # sort by rank and return
+      @records.sort { |a,b| b.rank <=> a.rank } # highest to lowest
+    end
+  
+    private
+  
+    # ranks from 0 to 100
+    def cosine_rank(vector1, vector2)
+      (cosine(vector1, vector2) + 1) / 2 * 100
+    end
+  
+    # see http://www.ltcconline.net/greenl/courses/107/vectors/DOTCROS.HTM
+    # and http://ruby-doc.org/stdlib/libdoc/matrix/rdoc/index.html
+    # will be in range -1 to 1
+    def cosine(vector1, vector2)
+      dot_product = vector1.inner_product(vector2)
+      dot_product / (vector1.r * vector2.r) # Vector#r is same as ||v||
+    end
+  
+    def make_query_vector(query)
+      make_vector(query, true)
+    end
+  
+    # NOTE: will choke if string contains words not in vocab
+    #       this is why, when we make the query vector, we do an
+    #       intersection of tokens with the vocab
+    def make_vector(string, ensure_words_in_vocab = false)
+      @vector_cache = {}
+      @vector_cache[string] ||= begin
+        arr = Array.new(vector_keyword_index.size, 0)
+
+        # uses tf*idf (http://en.wikipedia.org/wiki/Tf-idf)
+        words = tokenize(string)
+        words &= @vocab if ensure_words_in_vocab
+        words.uniq.each do |word| 
+          tf = count_in_array(words, word)
+          idf = @documents.size / count_in_array(@documents, proc { |doc| tokenize(doc).include?(word) })
+
+          index = vector_keyword_index[word]
+          arr[index] = tf * idf
+        end
+      
+        Vector.elements(arr, false)
+      end
+    end
+  
+    def vector_keyword_index
+      @vector_keyword_index ||= begin
+        index, offset = {}, 0      
+      
+        @vocab.each do |keyword|
+          index[keyword] = offset
+          offset += 1
+        end
+      
+        index
+      end
+    end
+  
+    def tokenize(string)
+      @tokenize_cache ||= {}
+      @tokenize_cache[string] ||= Tokenizer.tokenize(string)
+    end
+  
+    # could use Array#count, but 1.8.6 on Heroku don't have it only 1.8.7 >
+    def count_in_array(array, item)
+      count = 0
+      if item.is_a? Proc
+        array.each { |i| count += 1 if item.call(i) }
+      else
+        array.each { |i| count += 1 if i == item }
+      end    
+      count
+    end
+  end
+end

data/lib/vss/tokenizer.rb

@@ -0,0 +1,17 @@
+require "stemmer"
+
+module VSS
+  class Tokenizer
+    STOP_WORDS = %w[
+      a b c d e f g h i j k l m n o p q r s t u v w x y z
+      an and are as at be by for from has he in is it its
+      of on that the to was were will with upon without among
+    ]
+  
+    def self.tokenize(string)
+      stripped = string.to_s.gsub(/[^a-z0-9\-\s\']/i, "") # remove punctuation
+      words = stripped.split(/\s+/).reject(&:blank?).map(&:downcase).map(&:stem)
+      words.reject { |word| STOP_WORDS.include?(word) }.uniq
+    end
+  end
+end

data/lib/vss.rb

@@ -0,0 +1 @@
+require "vss/engine"

data/vss.gemspec

@@ -0,0 +1,33 @@
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{vss}
+  s.version = "0.1.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Mark Dodwell"]
+  s.date = %q{2010-03-10}
+  s.description = %q{Simple vector space search engine}
+  s.email = %q{labs@mkdynamic.co.uk}
+  s.extra_rdoc_files = ["LICENSE", "README.md", "lib/vss.rb", "lib/vss/engine.rb", "lib/vss/tokenizer.rb"]
+  s.files = ["LICENSE", "Manifest", "README.md", "Rakefile", "lib/vss.rb", "lib/vss/engine.rb", "lib/vss/tokenizer.rb", "vss.gemspec"]
+  s.homepage = %q{http://github.com/mkdynamic/vss}
+  s.rdoc_options = ["--line-numbers", "--inline-source", "--title", "Vss", "--main", "README.md"]
+  s.require_paths = ["lib"]
+  s.rubyforge_project = %q{vss}
+  s.rubygems_version = %q{1.3.5}
+  s.summary = %q{Simple vector space search engine}
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<stemmer>, [">= 0", "= 1.0.1"])
+    else
+      s.add_dependency(%q<stemmer>, [">= 0", "= 1.0.1"])
+    end
+  else
+    s.add_dependency(%q<stemmer>, [">= 0", "= 1.0.1"])
+  end
+end

metadata

@@ -0,0 +1,83 @@
+--- !ruby/object:Gem::Specification 
+name: vss
+version: !ruby/object:Gem::Version 
+  version: 0.1.0
+platform: ruby
+authors: 
+- Mark Dodwell
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-03-10 00:00:00 +00:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: stemmer
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    - - "="
+      - !ruby/object:Gem::Version 
+        version: 1.0.1
+    version: 
+description: Simple vector space search engine
+email: labs@mkdynamic.co.uk
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE
+- README.md
+- lib/vss.rb
+- lib/vss/engine.rb
+- lib/vss/tokenizer.rb
+files: 
+- LICENSE
+- Manifest
+- README.md
+- Rakefile
+- lib/vss.rb
+- lib/vss/engine.rb
+- lib/vss/tokenizer.rb
+- vss.gemspec
+has_rdoc: true
+homepage: http://github.com/mkdynamic/vss
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --line-numbers
+- --inline-source
+- --title
+- Vss
+- --main
+- README.md
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "1.2"
+  version: 
+requirements: []
+
+rubyforge_project: vss
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 3
+summary: Simple vector space search engine
+test_files: []
+