text 0.2.0 → 1.0.0

data/README.rdoc

@@ -2,35 +2,54 @@
 
 A collection of text algorithms.
 
-= Usage
+== Usage
 
   require 'text'
 
-  font = Text::Figlet::Font.new('big.flf')
-  figlet = Text::Figlet::Typesetter.new(font)
-  figlet['Hello World']                         # => '...'
+=== Levenshtein distance
 
-  Text::Levenshtein.distance('test', 'test')    # => 0
-  Text::Levenshtein.distance('test', 'tent')    # => 1
+  Text::Levenshtein.distance('test', 'test')
+  # => 0
+  Text::Levenshtein.distance('test', 'tent')
+  # => 1
 
-  Text::Metaphone.metaphone('BRIAN')            # => 'BRN'
-  Text::Metaphone.double_metaphone('Coburn')    # => ['KPRN', nil]
-  Text::Metaphone.double_metaphone('Angier')    # => ['ANJ', 'ANJR']
+=== Metaphone
 
-  Text::Soundex.soundex('Knuth')                # => 'K530'
+  Text::Metaphone.metaphone('BRIAN')
+  # => 'BRN'
 
-  Text::PorterStemming.stem('abatements')       # => 'abat'
+  Text::Metaphone.double_metaphone('Coburn')
+  # => ['KPRN', nil]
+  Text::Metaphone.double_metaphone('Angier')
+  # => ['ANJ', 'ANJR']
 
-= Ruby 1.9 Compatibility
+=== Soundex
 
-Most parts of the library are now compatible including
-tests. The big exception are the Figlet libraries which
-allow you to quickly do text-rendering in ASCII.
-On 1.9, Figlet isn't loaded and isn't tested. 
+  Text::Soundex.soundex('Knuth')
+  # => 'K530'
 
-Work to integrate in Ruby 1.9 compatibility was done by
-Hampton Catlin (hcatlin)
+=== Porter stemming
 
-= License
+  Text::PorterStemming.stem('abatements')  # => 'abat'
+
+=== White similarity
+
+  white = Text::WhiteSimilarity.new
+  white.similarity('Healed', 'Sealed')   # 0.8
+  white.similarity('Healed', 'Help')     # 0.25
+
+Note that some intermediate information is cached on the instance to improve
+performance.
+
+== Ruby version compatibility
+
+The library has been tested on Ruby 1.8.6 to 1.9.3 and on JRuby.
+
+== Thanks
+
+* Hampton Catlin (hcatlin) for Ruby 1.9 compatibility work
+* Wilker Lúcio for the initial implementation of the White algorithm
+
+== License
 
 Same as Ruby.

data/Rakefile

@@ -1,48 +1,8 @@
 require 'rake'
 require 'rake/testtask'
-require 'rake/packagetask'
-require 'rake/gempackagetask'
-require 'rcov/rcovtask'
-require 'rake/rdoctask'
-
-$:.unshift(File.dirname(__FILE__) + '/lib')
-require 'text/version'
-
-gemspec = Gem::Specification.new do |s|
-  s.name              = 'text'
-  s.version           = Text::VERSION::STRING
-  s.summary           = 'A collection of text algorithms'
-  s.description       = 'A collection of text algorithms: Levenshtein, Soundex, Metaphone, Double Metaphone, Figlet, Porter Stemming'
-  s.files             = FileList['{lib,test}/**/*', 'README.rdoc', 'Rakefile']
-  s.require_path      = 'lib'
-  s.has_rdoc          = true
-  s.extra_rdoc_files  = %w[README.rdoc]
-  s.rubyforge_project = 'text'
-  s.homepage          = 'http://github.com/threedaymonk/text'
-  s.authors           = ['Paul Battley', 'Michael Neumann', 'Tim Fletcher']
-  s.email             = "pbattley@gmail.com"
-end
-
-Rake::GemPackageTask.new(gemspec) do |pkg|
-  pkg.need_tar_gz = true
-end
-
-Rake::PackageTask.new(gemspec.name, gemspec.version) do |pkg|
-  pkg.need_tar_gz = true
-  pkg.package_files.include gemspec.files
-end
 
 Rake::TestTask.new do |t|
   t.verbose = false
 end
 
-Rcov::RcovTask.new do |t|
-  t.rcov_opts = []
-end
-
-Rake::RDocTask.new do |t|
-  t.main = 'README.rdoc'
-  t.rdoc_files.include 'README.rdoc', 'lib/**/*.rb'
-end
-
 task :default => :test

data/lib/text/version.rb

@@ -1,7 +1,7 @@
 module Text
   module VERSION #:nodoc:
-    MAJOR = 0
-    MINOR = 2
+    MAJOR = 1
+    MINOR = 0
     TINY  = 0
 
     STRING = [MAJOR, MINOR, TINY].join('.')

data/lib/text/white_similarity.rb

@@ -0,0 +1,57 @@
+# encoding: utf-8
+# Original author: Wilker Lúcio <wilkerlucio@gmail.com>
+
+require "set"
+
+module Text
+
+  # Ruby implementation of the string similarity described by Simon White
+  # at: http://www.catalysoft.com/articles/StrikeAMatch.html
+  #
+  #                        2 * |pairs(s1) INTERSECT pairs(s2)|
+  #   similarity(s1, s2) = -----------------------------------
+  #                            |pairs(s1)| + |pairs(s2)|
+  #
+  # e.g.
+  #                                             2 * |{FR, NC}|
+  #   similarity(FRANCE, FRENCH) = ---------------------------------------
+  #                                |{FR,RA,AN,NC,CE}| + |{FR,RE,EN,NC,CH}|
+  #
+  #                              = (2 * 2) / (5 + 5)
+  #
+  #                              = 0.4
+  #
+  #   WhiteSimilarity.new.similarity("FRANCE", "FRENCH")
+  #
+  class WhiteSimilarity
+
+    def self.similarity(str1, str2)
+      new.similarity(str1, str2)
+    end
+
+    def initialize
+      @word_letter_pairs = {}
+    end
+
+    def similarity(str1, str2)
+      pairs1 = word_letter_pairs(str1)
+      pairs2 = word_letter_pairs(str2)
+
+      intersection = pairs1.inject(0) { |acc, pair|
+        pairs2.include?(pair) ? acc + 1 : acc
+      }
+      union = pairs1.length + pairs2.length
+
+      (2.0 * intersection) / union
+    end
+
+  private
+    def word_letter_pairs(str)
+      @word_letter_pairs[str] ||= Set.new(
+        str.upcase.split(/\s+/).map{ |word|
+          (0 ... (word.length - 1)).map { |i| str[i, 2] }
+        }.flatten
+      )
+    end
+  end
+end

data/test/test_white_similarity.rb

@@ -0,0 +1,29 @@
+require File.join(File.dirname(__FILE__), 'preamble')
+require "text/white_similarity"
+
+class WhiteSimilarityTest < Test::Unit::TestCase
+
+  def test_similarity
+    word = "Healed"
+
+    assert_in_delta 0.8,  Text::WhiteSimilarity.similarity(word, "Sealed"),  0.01
+    assert_in_delta 0.55, Text::WhiteSimilarity.similarity(word, "Healthy"), 0.01
+    assert_in_delta 0.44, Text::WhiteSimilarity.similarity(word, "Heard"),   0.01
+    assert_in_delta 0.40, Text::WhiteSimilarity.similarity(word, "Herded"),  0.01
+    assert_in_delta 0.25, Text::WhiteSimilarity.similarity(word, "Help"),    0.01
+    assert_in_delta 0.0,  Text::WhiteSimilarity.similarity(word, "Sold"),    0.01
+  end
+
+  def test_similarity_with_caching
+    word = "Healed"
+
+    white = Text::WhiteSimilarity.new
+
+    assert_in_delta 0.8,  white.similarity(word, "Sealed"),  0.01
+    assert_in_delta 0.55, white.similarity(word, "Healthy"), 0.01
+    assert_in_delta 0.44, white.similarity(word, "Heard"),   0.01
+    assert_in_delta 0.40, white.similarity(word, "Herded"),  0.01
+    assert_in_delta 0.25, white.similarity(word, "Help"),    0.01
+    assert_in_delta 0.0,  white.similarity(word, "Sold"),    0.01
+  end
+end

metadata

@@ -1,81 +1,75 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: text
-version: !ruby/object:Gem::Version 
-  version: 0.2.0
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+  prerelease: 
 platform: ruby
-authors: 
+authors:
 - Paul Battley
 - Michael Neumann
 - Tim Fletcher
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2010-03-03 00:00:00 +00:00
-default_executable: 
+date: 2011-11-21 00:00:00.000000000Z
 dependencies: []
-
-description: "A collection of text algorithms: Levenshtein, Soundex, Metaphone, Double Metaphone, Figlet, Porter Stemming"
+description: ! 'A collection of text algorithms: Levenshtein, Soundex, Metaphone,
+  Double Metaphone, Figlet, Porter Stemming'
 email: pbattley@gmail.com
 executables: []
-
 extensions: []
-
-extra_rdoc_files: 
+extra_rdoc_files:
 - README.rdoc
-files: 
-- lib/text/double_metaphone.rb
+files:
 - lib/text/levenshtein.rb
 - lib/text/metaphone.rb
-- lib/text/porter_stemming.rb
 - lib/text/soundex.rb
+- lib/text/white_similarity.rb
+- lib/text/double_metaphone.rb
+- lib/text/porter_stemming.rb
 - lib/text/util.rb
 - lib/text/version.rb
 - lib/text.rb
-- test/data/big.flf
-- test/data/big.txt
+- test/test_porter_stemming.rb
 - test/data/chunky.flf
-- test/data/chunky.txt
-- test/data/double_metaphone.csv
-- test/data/metaphone.txt
-- test/data/metaphone_buggy.txt
 - test/data/porter_stemming_input.txt
+- test/data/metaphone.txt
+- test/data/double_metaphone.csv
+- test/data/big.flf
 - test/data/porter_stemming_output.txt
+- test/data/metaphone_buggy.txt
+- test/data/chunky.txt
+- test/data/big.txt
 - test/preamble.rb
-- test/test_double_metaphone.rb
-- test/test_levenshtein.rb
-- test/test_metaphone.rb
-- test/test_porter_stemming.rb
 - test/test_soundex.rb
+- test/test_white_similarity.rb
+- test/test_metaphone.rb
+- test/test_levenshtein.rb
+- test/test_double_metaphone.rb
 - README.rdoc
 - Rakefile
-has_rdoc: true
 homepage: http://github.com/threedaymonk/text
 licenses: []
-
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      version: "0"
-  version: 
-required_rubygems_version: !ruby/object:Gem::Requirement 
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      version: "0"
-  version: 
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
 rubyforge_project: text
-rubygems_version: 1.3.5
+rubygems_version: 1.8.11
 signing_key: 
 specification_version: 3
 summary: A collection of text algorithms
 test_files: []
-