vss 0.1.4 → 0.1.8

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*

data/Gemfile

@@ -0,0 +1,2 @@
+source 'http://rubygems.org'
+gemspec

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2010 Mark Dodwell, mkdynamic
+Copyright (c) 2011 Mark Dodwell, mkdynamic
  
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the
@@ -17,4 +17,4 @@ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
-SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -4,7 +4,9 @@ A simple vector space search engine with **tf*idf** ranking.
 
 [More info, and details of how it works.](http://madeofcode.com/posts/69-vss-a-vector-space-search-engine-in-ruby)
 
-NB. Version 0.1.4 and up requires Ruby 1.9.2.
+## Requirements
+
+Ruby >= 1.8.7. Tested in (MRI 1.8.7 + 1.9.2).
 
 ## Install
 
@@ -23,7 +25,7 @@ To perform a search on a collection of documents:
     
 ## Rails/ActiveRecord
 
-If you want to search a collection of `ActiveRecord` objects, you need to pass a **documentizer** `proc` when initializing `VSS::Engine` which will convert the objects into documents (which are simply strings). For example:
+If you want to search a collection of `ActiveRecord` objects, you need to pass a **documentizer** `Proc` when initializing `VSS::Engine` which will convert the objects into documents (which are simply strings). For example:
 
     class Page < ActiveRecord::Base
         #attrs: title, content
@@ -41,5 +43,4 @@ This isn't designed to be used on huge collections of records. The original use
 
 Heavily inspired by [Joesph Wilk's article on building a vector space search engine in Python](http://blog.josephwilk.net/projects/building-a-vector-space-search-engine-in-python.html).
 
-Written by Mark Dodwell
-([Design & Code](http://madeofcode.com))
+Written by Mark Dodwell ([@madeofcode](http://twitter.com/madeofcode))

data/Rakefile

@@ -1,9 +1,4 @@
-require "echoe"  
-  
-Echoe.new("vss", "0.1.4") do |p|  
-  p.description = "Simple vector space search engine"    
-  p.url = "http://github.com/mkdynamic/vss"   
-  p.author = "Mark Dodwell"      
-  p.email = "labs@mkdynamic.co.uk"     
-  p.runtime_dependencies = ["stemmer >=1.0.1", "activesupport >=3.0.0"]
-end  
+require "rubygems"
+require "bundler/setup"
+
+Bundler::GemHelper.install_tasks

data/lib/vss/engine.rb

@@ -1,5 +1,5 @@
-require "matrix"
-require "vss/tokenizer"
+require 'matrix'
+require 'vss/tokenizer'
 
 module VSS
   class Engine    
@@ -32,7 +32,13 @@ module VSS
   
     # ranks from 0 to 100
     def cosine_rank(vector1, vector2)
-      cosine(vector1, vector2) / 1 * 100
+      cos = cosine(vector1, vector2)
+      
+      if cos > 0
+        cos / 1 * 100
+      else
+        0
+      end
     end
   
     # see http://www.ltcconline.net/greenl/courses/107/vectors/DOTCROS.HTM
@@ -67,12 +73,11 @@ module VSS
     end
     
     def tf(token, tokens)
-      count_in_array(tokens, token)
+      tokens.count { |t| t == token }
     end
     
     def idf(token, docs)
-      docs_with_token_count = count_in_array(docs, proc { |doc| tokenize(doc).include?(token) })
-      docs.size / docs_with_token_count     
+      docs.size / docs.count { |d| tokenize(d).include?(token) }     
     end
     
     # http://en.wikipedia.org/wiki/Tf-idf
@@ -97,16 +102,5 @@ module VSS
       @tokenize_cache ||= {}
       @tokenize_cache[string] ||= Tokenizer.tokenize(string)
     end
-  
-    # could use Array#count, but only for Ruby 1.8.7 >=
-    def count_in_array(array, item)
-      count = 0
-      if item.is_a? Proc
-        array.each { |i| count += 1 if item.call(i) }
-      else
-        array.each { |i| count += 1 if i == item }
-      end    
-      count
-    end
   end
-end
+end

data/lib/vss/tokenizer.rb

@@ -1,5 +1,4 @@
-require "stemmer"
-require "active_support/core_ext"
+require 'stemmer'
 
 module VSS
   class Tokenizer
@@ -7,12 +6,12 @@ module VSS
       a b c d e f g h i j k l m n o p q r s t u v w x y z
       an and are as at be by for from has he in is it its
       of on that the to was were will with upon without among
-    ]
+    ].inject({}) { |h,v| h[v] = true; h }
   
     def self.tokenize(string)
-      stripped = string.to_s.gsub(/[^a-z0-9\-\s\']/i, "") # remove punctuation
-      words = stripped.split(/\s+/).reject(&:blank?).map(&:downcase).map(&:stem)
-      words.reject { |word| STOP_WORDS.include?(word) }.uniq
+      stripped = string.to_s.gsub(/[^a-z0-9\-\s\']/i, "") # removes punctuation
+      words = stripped.split(/\s+/).reject { |word| word.match(/^\s*$/) }.map(&:downcase).map(&:stem)
+      words.reject { |word| STOP_WORDS.key?(word) }.uniq
     end
   end
-end
+end

data/lib/vss/version.rb

@@ -0,0 +1,3 @@
+module VSS
+  VERSION = '0.1.8'
+end

data/lib/vss.rb

@@ -1 +1,2 @@
-require "vss/engine"
+require 'vss/version'
+require 'vss/engine'

data/test/{search_test.rb → test.rb}

@@ -1,7 +1,7 @@
 require 'test/unit'
 require 'vss'
 
-class SearchTest < Test::Unit::TestCase
+class VSSTest < Test::Unit::TestCase
   def setup
     @doc1 = "I'm not even going to mention any TV series."
     @doc2 = "The Wire is the best thing ever. Fact."
@@ -27,9 +27,22 @@ class SearchTest < Test::Unit::TestCase
   
   def test_ranking
     results = @engine.search("How can you compare The Wire with Lost?")
-    assert_equal 82.17814036133181, results[0].rank
-    assert_equal 3.0816677568068283, results[1].rank
-    assert_equal 1.3798683116522041, results[2].rank
-    assert_equal 0.8753091481365445, results[3].rank
+    assert_similar_float 82.1781, results[0].rank
+    assert_similar_float 3.08166, results[1].rank
+    assert_similar_float 1.37986, results[2].rank
+    assert_similar_float 0.87530, results[3].rank
+  end
+  
+  def test_no_match
+    results = @engine.search("Zebra funnels cash")
+    assert_equal 0, results.size
+  end
+  
+private
+
+  def assert_similar_float(expected, actual, msg = nil)
+    assert Float === expected, "not a Float"
+    sig_figs = [10, actual.to_s.size, expected.to_s.size].min - 1
+    assert_equal expected.to_s[0, sig_figs], actual.to_s[0, sig_figs], msg
   end
 end

data/vss.gemspec

@@ -1,36 +1,22 @@
-# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/vss/version', __FILE__)
 
 Gem::Specification.new do |s|
-  s.name = %q{vss}
-  s.version = "0.1.4"
-
-  s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
-  s.authors = [%q{Mark Dodwell}]
-  s.date = %q{2011-06-29}
-  s.description = %q{Simple vector space search engine}
-  s.email = %q{labs@mkdynamic.co.uk}
-  s.extra_rdoc_files = [%q{LICENSE}, %q{README.md}, %q{lib/vss.rb}, %q{lib/vss/engine.rb}, %q{lib/vss/tokenizer.rb}]
-  s.files = [%q{LICENSE}, %q{README.md}, %q{Rakefile}, %q{lib/vss.rb}, %q{lib/vss/engine.rb}, %q{lib/vss/tokenizer.rb}, %q{test/search_test.rb}, %q{Manifest}, %q{vss.gemspec}]
-  s.homepage = %q{http://github.com/mkdynamic/vss}
-  s.rdoc_options = [%q{--line-numbers}, %q{--inline-source}, %q{--title}, %q{Vss}, %q{--main}, %q{README.md}]
-  s.require_paths = [%q{lib}]
-  s.rubyforge_project = %q{vss}
-  s.rubygems_version = %q{1.8.5}
-  s.summary = %q{Simple vector space search engine}
-  s.test_files = [%q{test/search_test.rb}]
-
-  if s.respond_to? :specification_version then
-    s.specification_version = 3
-
-    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
-      s.add_runtime_dependency(%q<stemmer>, [">= 1.0.1"])
-      s.add_runtime_dependency(%q<activesupport>, [">= 3.0.0"])
-    else
-      s.add_dependency(%q<stemmer>, [">= 1.0.1"])
-      s.add_dependency(%q<activesupport>, [">= 3.0.0"])
-    end
-  else
-    s.add_dependency(%q<stemmer>, [">= 1.0.1"])
-    s.add_dependency(%q<activesupport>, [">= 3.0.0"])
-  end
+  s.name        = "vss"
+  s.version     = VSS::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Mark Dodwell"]
+  s.email       = ["labs@mkdynamic.co.uk"]
+  s.homepage    = "https://github.com/mkdynamic/vss"
+  s.summary     = "Vector Space Search"
+  s.description = "A simple vector space search engine with tf*idf ranking."
+  
+  s.required_ruby_version = ">= 1.8.7"
+  s.add_development_dependency "bundler"
+  s.add_development_dependency "rake", "0.8.7"
+  s.add_runtime_dependency "stemmer"
+  
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- test/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
 end

metadata

@@ -1,8 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: vss
 version: !ruby/object:Gem::Version 
-  prerelease: 
-  version: 0.1.4
+  hash: 11
+  prerelease: false
+  segments: 
+  - 0
+  - 1
+  - 8
+  version: 0.1.8
 platform: ruby
 authors: 
 - Mark Dodwell
@@ -10,63 +15,81 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-06-29 00:00:00 Z
+date: 2011-07-05 00:00:00 -07:00
+default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
-  name: stemmer
-  prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
+  version_requirements: &id001 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        version: 1.0.1
-  type: :runtime
-  version_requirements: *id001
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  requirement: *id001
+  type: :development
+  name: bundler
+  prerelease: false
 - !ruby/object:Gem::Dependency 
-  name: activesupport
+  version_requirements: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - "="
+      - !ruby/object:Gem::Version 
+        hash: 49
+        segments: 
+        - 0
+        - 8
+        - 7
+        version: 0.8.7
+  requirement: *id002
+  type: :development
+  name: rake
   prerelease: false
-  requirement: &id002 !ruby/object:Gem::Requirement 
+- !ruby/object:Gem::Dependency 
+  version_requirements: &id003 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        version: 3.0.0
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  requirement: *id003
   type: :runtime
-  version_requirements: *id002
-description: Simple vector space search engine
-email: labs@mkdynamic.co.uk
+  name: stemmer
+  prerelease: false
+description: A simple vector space search engine with tf*idf ranking.
+email: 
+- labs@mkdynamic.co.uk
 executables: []
 
 extensions: []
 
-extra_rdoc_files: 
-- LICENSE
-- README.md
-- lib/vss.rb
-- lib/vss/engine.rb
-- lib/vss/tokenizer.rb
+extra_rdoc_files: []
+
 files: 
+- .gitignore
+- Gemfile
 - LICENSE
 - README.md
 - Rakefile
 - lib/vss.rb
 - lib/vss/engine.rb
 - lib/vss/tokenizer.rb
-- test/search_test.rb
-- Manifest
+- lib/vss/version.rb
+- test/test.rb
 - vss.gemspec
-homepage: http://github.com/mkdynamic/vss
+has_rdoc: true
+homepage: https://github.com/mkdynamic/vss
 licenses: []
 
 post_install_message: 
-rdoc_options: 
-- --line-numbers
-- --inline-source
-- --title
-- Vss
-- --main
-- README.md
+rdoc_options: []
+
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
@@ -74,19 +97,27 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      version: "0"
+      hash: 57
+      segments: 
+      - 1
+      - 8
+      - 7
+      version: 1.8.7
 required_rubygems_version: !ruby/object:Gem::Requirement 
   none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      version: "1.2"
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
 requirements: []
 
-rubyforge_project: vss
-rubygems_version: 1.8.5
+rubyforge_project: 
+rubygems_version: 1.3.7
 signing_key: 
 specification_version: 3
-summary: Simple vector space search engine
+summary: Vector Space Search
 test_files: 
-- test/search_test.rb
+- test/test.rb

data/Manifest

@@ -1,8 +0,0 @@
-LICENSE
-README.md
-Rakefile
-lib/vss.rb
-lib/vss/engine.rb
-lib/vss/tokenizer.rb
-test/search_test.rb
-Manifest