simple_similarity 0.0.1

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in simple_similarity.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Ori Pekelman
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,79 @@
+# SimpleSimilarity
+
+This is an implemenation of a very trivial, and very costly algorithm to find
+Similarity in a collection of items. It calculates a  distance between the
+length of the items compressed each on its own, and compressed when concatenated
+The theory is sound.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'simple_similarity'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install simple_similarity
+
+## Usage
+
+Get the distance between "Hello" and "World", always takes to arguments.
+
+    SimpleSimilarity::SimpleSimilarity.distance("Hello","World")
+
+    $ 0.125
+
+    SimpleSimilarity::SimpleSimilarity.case_insensitive_distance("Hello","hello")
+
+    $ 1.0
+
+Get a distance matrix, the first argument is an array of elements, the second
+is a cutoff ratio for the detected similarities (0.5 is quite similar, above
+0.8 we are  basically identical, depending on the length of the original strings..)
+
+     SimpleSimilarity::SimpleSimilarity.distance_matrix(%w( hello, hell, world), 0.5, true, false)
+
+     $  => {
+     :matrix=>[
+     [nil, 0.5833333333333333, 0.11764705882352944], 
+     [nil, nil, 0.125], 
+     [nil, nil, nil]
+     ], 
+     :similar_terms=>[{
+     :first_term=>[0, "hello,"], 
+     :second_term=>[1, "hell,"], 
+     :score=>0.5833333333333333}]} 
+
+### Caveat : 
+1. this algorithm costs (n^2/2) - n in time and n^2 in memory
+>So if n= 100,000 
+> there will be 4,999,900,000 calculations
+2. setting only_hits to true does not return the matrix only the hits but optimizes memory usage.
+### Note:
+the matrix can be used later for clustering....     
+
+#Testing
+spec:
+    
+    bundle exec rspec spec         
+
+
+Benchmark:
+
+On my machine comparing 100 items takes around 8 seconds: 
+    
+    bundle exec ruby spec/benchmark.rb
+
+If what you want to compare are longer documents, swap the 
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/lib/simple_similarity.rb

@@ -0,0 +1,78 @@
+require "simple_similarity/version"
+require "zlib"
+
+module SimpleSimilarity
+  class SimpleSimilarity
+    #calculate similarity by comparing the length of compressed(a) + length of
+    #compressed (b) to the lengt of compressed (a and b concatenated)
+    def self.distance(a, b)
+      a = a.to_s
+      b = b.to_s
+
+      # optimization: well if they are equal lets not do expensive stuff
+      if a==b then return 1.0 end
+      
+      # optimization: if one is empty and the other not they are totally different
+      if ((a.empty? && !b.empty?) || (b.empty? && !a.empty?)) then return (0.to_f)  end
+        
+      # This is hacky, but useful. When strings are too short the compression
+      # is not sufficient.
+      if (a.length < 40 || b.length < 40)
+        a = a * 10
+        b = b * 10
+      end
+  
+    # 8 characters is the minimal zlib compressed stream, this is basically a constant length we remove
+     minimal_zip_length = 7 
+     azip = Zlib.deflate(a).length - minimal_zip_length 
+     bzip = Zlib.deflate(b).length - minimal_zip_length
+   
+     aplusbzip    =    (Zlib.deflate(a + b).length) - minimal_zip_length
+     ((azip + bzip).to_f/aplusbzip.to_f) - 1 # -1 is because something that is totally different will get 1.00 totally the same 2.00. 
+    end
+  
+    #Normalize the string, downcase remove non alphanumeric characters
+    def self.case_insensitive_distance(a, b)
+       self.distance(a.downcase.gsub(/[^[:alnum:]]/, ' '), b.downcase.gsub(/[^[:alnum:]]/, ' '))
+    end
+  
+    # This method will return a distance matrix as well as matches beyond a supplies threshold
+    # Caveat : this algorithm costs (n^2/2) - n in time and n^2 in memory
+    # So if n= 100,000 
+    # there will be 4,999,900,000 calculations
+    # setting only_hits to true does not return the matrix only the hits
+    # the matrix can be used later for clustering....
+  
+    #        "hello",	"hello",	"world", 	"hello world", 	"hello cruel world"
+    # "hello"	[nil, 	1.0, 	0.125, 	0.41176470588235303, 	0.28],
+    # "hello"	[nil, 	nil, 	0.125, 	0.41176470588235303, 	0.28],
+    # "world"	[nil, 	nil, 	nil, 	0.41176470588235303, 	0.28],
+    # "hello world"	[nil, 	nil, 	nil, 	nil, 	0.52],
+    # "hello cruel world"	[nil, 	nil, 	nil, 	nil, 	nil]
+    def self.distance_matrix(arr, cutoff = 0.5, case_insensitive = false, only_hits = true )
+      sims =[]
+      len = arr.length
+      if only_hits 
+        mat = []
+      else 
+        mat = Array.new(len){Array.new(len)}
+      end
+      
+      dist = 0.to_f
+      for i in 0..len
+        for j in i+1..len -1
+        if case_insensitive then
+          dist = self.case_insensitive_distance(arr[i], arr[j])
+        else
+          dist = self.distance(arr[i], arr[j])
+        end
+        if dist >= cutoff
+          sims.push({:first_term=>[i,arr[i]],:second_term=>[j,arr[j]],:score=>dist})
+        end
+        mat[i][j] = dist unless only_hits
+       end
+      end  
+      {:matrix=>mat,:similar_terms=>sims}
+    end
+  end
+end

data/lib/simple_similarity/version.rb

@@ -0,0 +1,3 @@
+module SimpleSimilarity
+  VERSION = "0.0.1"
+end

data/simple_similarity.gemspec

@@ -0,0 +1,23 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'simple_similarity/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "simple_similarity"
+  gem.version       = SimpleSimilarity::VERSION
+  gem.authors       = ["Ori Pekelman"]
+  gem.email         = ["ori@pekelman.com"]
+  gem.description   = %q{A simple (slow but beautiful) algorithm to find similar things}
+  gem.summary       = %q{A simple (slow but beautiful) algorithm to find similar things}
+  gem.homepage      = "https://github.com/OriPekelman/simple_similarity"
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+  gem.add_development_dependency "rspec", "~> 2.6"
+  gem.add_development_dependency "pry"
+  gem.add_development_dependency "awesome_print"
+  gem.add_development_dependency "colorize"
+end

data/spec/benchmark.rb

@@ -0,0 +1,18 @@
+require 'simple_similarity'
+require "benchmark"
+ 
+a = []
+CHARS = (?a..?z).to_a + ["e","e","e","e","e","a","a","a","a","a","a","a","a","a","a","a","a"]
+1000.times {a.push(5.times.inject("") {|s, i| s << CHARS[rand(CHARS.size)]})}
+
+time = Benchmark.measure do
+  puts SimpleSimilarity::SimpleSimilarity.distance_matrix(a, 0.5, false, true)[:similar_terms]
+end
+puts "Without matrix"
+puts time
+
+time = Benchmark.measure do
+  puts SimpleSimilarity::SimpleSimilarity.distance_matrix(a, 0.5, false, false)[:similar_terms]
+end
+puts "With matrix"
+puts time

data/spec/simple_similarity_spec.rb

@@ -0,0 +1,41 @@
+require 'simple_similarity'
+
+describe SimpleSimilarity::SimpleSimilarity do
+  it "Hello is identical to Hello" do
+    SimpleSimilarity::SimpleSimilarity.distance("Hello","Hello").should eql(1.0)
+  end
+
+  it "Hello is not similar to world" do
+    SimpleSimilarity::SimpleSimilarity.distance("Hello","World").should eql(0.125)
+  end
+  
+  it "Hello is somewhat similar hello" do
+    SimpleSimilarity::SimpleSimilarity.distance("Hello","hello").should eql(0.5)
+  end
+  
+  it "Hello is identical to hello when case insensitive" do
+    SimpleSimilarity::SimpleSimilarity.case_insensitive_distance("Hello","hello").should eql(1.0)
+  end
+  
+  it "from : hello, Hello, hell, world, cruel, crude with a cutoff of 0.5 we should have two couples" do
+    SimpleSimilarity::SimpleSimilarity.distance_matrix(%w( hello, Hello, hell, world, cruel, crud), 0.5)[:similar_terms].length.should eql(2)
+  end
+
+  it "from : hello, Hello, hell, world, cruel, crude with a cutoff of 0.5 we should have three couples case insensitive" do
+    SimpleSimilarity::SimpleSimilarity.distance_matrix(%w( hello, Hello, hell, world, cruel, crud), 0.5, true)[:similar_terms].length.should eql(3)
+  end
+  
+  it "from : hello world, Hello world, hell world, world, cruel, crude with a cutoff of 1.0 we should have seven couples case insensitive" do
+    SimpleSimilarity::SimpleSimilarity.distance_matrix(%w( hello world, Hello world, hell world, world, cruel, crude), 1.0, true)[:similar_terms].length.should eql(7)
+  end
+  
+  it "setting only_hits to false gives back a matrix" do
+    SimpleSimilarity::SimpleSimilarity.distance_matrix(%w( hello, world), 1.0, true, false)[:matrix].length.should eql(2)
+  end
+
+  it "by default does not gives an empty matrix" do
+    SimpleSimilarity::SimpleSimilarity.distance_matrix(%w( hello, world))[:matrix].length.should eql(0)
+  end
+  
+  
+end

metadata

@@ -0,0 +1,122 @@
+--- !ruby/object:Gem::Specification
+name: simple_similarity
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Ori Pekelman
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-01-08 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.6'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: awesome_print
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: colorize
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: A simple (slow but beautiful) algorithm to find similar things
+email:
+- ori@pekelman.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/simple_similarity.rb
+- lib/simple_similarity/version.rb
+- simple_similarity.gemspec
+- spec/benchmark.rb
+- spec/simple_similarity_spec.rb
+homepage: https://github.com/OriPekelman/simple_similarity
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: A simple (slow but beautiful) algorithm to find similar things
+test_files:
+- spec/benchmark.rb
+- spec/simple_similarity_spec.rb
+has_rdoc: