strabo 0.0.0 → 0.0.2

data/VERSION

@@ -1 +1 @@
-0.0.0
+0.0.2

data/features/index.feature

@@ -61,4 +61,12 @@ Feature: Index a document
       | z | 3 |
       | a | 1 |
       | b | 1 |
-      | c | 1 |
+      | c | 1 |
+    Then it should have the following term set:
+      | T |
+      | x |
+      | y |
+      | z |
+      | a |
+      | b |
+      | c |

data/features/steps/index_steps.rb

@@ -34,4 +34,10 @@ Then /it should have the following "(.+)" term frequencies:/ do |attribute, tabl
     @context.keywords[attribute].keys.should include(row['term'])
     @context.keywords[attribute][row['term']].should eql(row['frequency'].to_i)
   end
-end
+end
+
+Then /it should have the following term set:/ do |table|
+  table.rows.each do |row|
+    @context.keywords(:as => :set).should include(row.first)
+  end
+end

data/lib/strabo.rb

@@ -1,4 +1,4 @@
-require 'Set'
+require 'set'
 
 # Strabo assists full text search indexing by generating term-frequency maps
 # for an object's attributes.  The term-frequency map may be flattened into
@@ -45,16 +45,35 @@ module Strabo
     # Invokes stemmer on token.  If no stemmer has been configured, it will
     # return the original token.
     #
-    # @param [String] token 
+    # @param [Array] one or more tokens 
     #
-    # @return [String] result of stemming
+    # @return [Array] stemmed tokens
     #
     # @see Strabo#stemmer
-    def self.stem(token)
-      @stemmer.nil? ? token : @stemmer.call(token)
+    def self.stem(*tokens)
+      tokens.map do |token|
+        @stemmer.nil? ? token : @stemmer.call(token)
+      end.flatten
     end
   end
   
+  # Defines how a single string is divided into multiple strings.
+  module Tokenizer
+    
+    # Break a string into a list of strings.
+    #
+    # @param [String] text to convert into a list
+    # @param [Regex] delimiter used to scan the string
+    #
+    # @return [Array] list of stemmed terms
+    #
+    # @private
+    def self.tokenize(value, delimiter = /\S+/)
+      value.downcase.gsub(/[^a-z0-9\s]/i,'').scan(delimiter)
+    end
+    
+  end
+  
   module Indexer
     
     # Get attribute-term-frequency map. If flattened, a term-frequency map
@@ -66,24 +85,17 @@ module Strabo
     #                { term => frequency } map.
     def keywords(flatten = false)
       @term_map = {}
-      self.each { |key, value| @term_map[key] = frequency(tokenize(value)) }
-      flatten ? flatten_keyword_map(@term_map) : @term_map
+      self.each { |key, value| @term_map[key] = frequency(Stemmer::stem(Tokenizer::tokenize(value))) }
+      
+      case flatten
+      when false : @term_map
+      when true  : flatten_keyword_map(@term_map)
+      else         flatten_keyword_map(@term_map).keys
+      end
     end
     
   private
   
-    # Break a string into a list of strings.
-    #
-    # @param [String] text to convert into a list
-    # @param [Regex] delimiter used to scan the string
-    #
-    # @return [Array] list of stemmed terms
-    #
-    # @private
-    def tokenize(value, delimiter = /\S+/)
-      value.downcase.scan(delimiter).map { |token| Strabo::Stemmer.stem(token) }
-    end
-    
     # Tally the number of occurrences of a value in a list.
     #
     # @param [Array] list of terms to count

data/readme.markdown

@@ -2,7 +2,7 @@
 
 ## About
 
-Strabo makes preparing a Ruby object for full text search by tokenizing an objects attributes.
+Strabo prepares a Ruby object for full text search by tokenizing an objects attributes.
 
     class Book < Hash
       include Strabo::Indexer

data/strabo.gemspec

@@ -0,0 +1,48 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{strabo}
+  s.version = "0.0.2"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Jon Morton"]
+  s.date = %q{2010-04-20}
+  s.description = %q{Simplified tokenization, stemming, and term-frequency map indexes}
+  s.email = %q{jon.morton@gmail.com }
+  s.files = [
+    ".gitignore",
+     "Rakefile",
+     "VERSION",
+     "examples/book.rb",
+     "features/index.feature",
+     "features/stemming.feature",
+     "features/steps/index_steps.rb",
+     "features/steps/stemmer_steps.rb",
+     "features/support/env.rb",
+     "lib/strabo.rb",
+     "readme.markdown",
+     "strabo.gemspec"
+  ]
+  s.homepage = %q{http://github.com/jmorton/strabo}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.6}
+  s.summary = %q{Full text search utilities for Ruby}
+  s.test_files = [
+    "examples/book.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+    else
+    end
+  else
+  end
+end
+

metadata

@@ -1,7 +1,12 @@
 --- !ruby/object:Gem::Specification 
 name: strabo
 version: !ruby/object:Gem::Version 
-  version: 0.0.0
+  prerelease: false
+  segments: 
+  - 0
+  - 0
+  - 2
+  version: 0.0.2
 platform: ruby
 authors: 
 - Jon Morton
@@ -9,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-03-28 00:00:00 -04:00
+date: 2010-04-20 00:00:00 -04:00
 default_executable: 
 dependencies: []
 
@@ -33,6 +38,7 @@ files:
 - features/support/env.rb
 - lib/strabo.rb
 - readme.markdown
+- strabo.gemspec
 has_rdoc: true
 homepage: http://github.com/jmorton/strabo
 licenses: []
@@ -46,18 +52,20 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      segments: 
+      - 0
       version: "0"
-  version: 
 required_rubygems_version: !ruby/object:Gem::Requirement 
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      segments: 
+      - 0
       version: "0"
-  version: 
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.3.5
+rubygems_version: 1.3.6
 signing_key: 
 specification_version: 3
 summary: Full text search utilities for Ruby